~hc/RK356X_SDK_RELEASE.git

..	..	@@ -55,6 +55,8 @@
55	55	#include "t4fw_api.h"
56	56	#include "cxgb4_ptp.h"
57	57	#include "cxgb4_uld.h"
	58	+#include "cxgb4_tc_mqprio.h"
	59	+#include "sched.h"
58	60
59	61	/*
60	62	* Rx buffer size. We use largish buffers if possible but settle for single
..	..	@@ -80,9 +82,10 @@
80	82	* Max number of Tx descriptors we clean up at a time. Should be modest as
81	83	* freeing skbs isn't cheap and it happens while holding locks. We just need
82	84	* to free packets faster than they arrive, we eventually catch up and keep
83		- * the amortized cost reasonable. Must be >= 2 * TXQ_STOP_THRES.
	85	+ * the amortized cost reasonable. Must be >= 2 * TXQ_STOP_THRES. It should
	86	+ * also match the CIDX Flush Threshold.
84	87	*/
85		-#define MAX_TX_RECLAIM 16
	88	+#define MAX_TX_RECLAIM 32
86	89
87	90	/*
88	91	* Max number of Rx buffers we replenish at a time. Again keep this modest,
..	..	@@ -268,7 +271,6 @@
268	271	}
269	272	EXPORT_SYMBOL(cxgb4_map_skb);
270	273
271		-#ifdef CONFIG_NEED_DMA_MAP_STATE
272	274	static void unmap_skb(struct device dev, const struct sk_buff skb,
273	275	const dma_addr_t *addr)
274	276	{
..	..	@@ -283,6 +285,7 @@
283	285	dma_unmap_page(dev, *addr++, skb_frag_size(fp), DMA_TO_DEVICE);
284	286	}
285	287
	288	+#ifdef CONFIG_NEED_DMA_MAP_STATE
286	289	/**
287	290	* deferred_unmap_destructor - unmap a packet when it is freed
288	291	* @skb: the packet
..	..	@@ -297,68 +300,9 @@
297	300	}
298	301	#endif
299	302
300		-static void unmap_sgl(struct device dev, const struct sk_buff skb,
301		- const struct ulptx_sgl sgl, const struct sge_txq q)
302		-{
303		- const struct ulptx_sge_pair *p;
304		- unsigned int nfrags = skb_shinfo(skb)->nr_frags;
305		-
306		- if (likely(skb_headlen(skb)))
307		- dma_unmap_single(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0),
308		- DMA_TO_DEVICE);
309		- else {
310		- dma_unmap_page(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0),
311		- DMA_TO_DEVICE);
312		- nfrags--;
313		- }
314		-
315		- /*
316		- * the complexity below is because of the possibility of a wrap-around
317		- * in the middle of an SGL
318		- */
319		- for (p = sgl->sge; nfrags >= 2; nfrags -= 2) {
320		- if (likely((u8 )(p + 1) <= (u8 )q->stat)) {
321		-unmap: dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
322		- ntohl(p->len[0]), DMA_TO_DEVICE);
323		- dma_unmap_page(dev, be64_to_cpu(p->addr[1]),
324		- ntohl(p->len[1]), DMA_TO_DEVICE);
325		- p++;
326		- } else if ((u8 )p == (u8 )q->stat) {
327		- p = (const struct ulptx_sge_pair *)q->desc;
328		- goto unmap;
329		- } else if ((u8 )p + 8 == (u8 )q->stat) {
330		- const __be64 addr = (const __be64 )q->desc;
331		-
332		- dma_unmap_page(dev, be64_to_cpu(addr[0]),
333		- ntohl(p->len[0]), DMA_TO_DEVICE);
334		- dma_unmap_page(dev, be64_to_cpu(addr[1]),
335		- ntohl(p->len[1]), DMA_TO_DEVICE);
336		- p = (const struct ulptx_sge_pair *)&addr[2];
337		- } else {
338		- const __be64 addr = (const __be64 )q->desc;
339		-
340		- dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
341		- ntohl(p->len[0]), DMA_TO_DEVICE);
342		- dma_unmap_page(dev, be64_to_cpu(addr[0]),
343		- ntohl(p->len[1]), DMA_TO_DEVICE);
344		- p = (const struct ulptx_sge_pair *)&addr[1];
345		- }
346		- }
347		- if (nfrags) {
348		- __be64 addr;
349		-
350		- if ((u8 )p == (u8 )q->stat)
351		- p = (const struct ulptx_sge_pair *)q->desc;
352		- addr = (u8 )p + 16 <= (u8 )q->stat ? p->addr[0] :
353		- (const __be64 )q->desc;
354		- dma_unmap_page(dev, be64_to_cpu(addr), ntohl(p->len[0]),
355		- DMA_TO_DEVICE);
356		- }
357		-}
358		-
359	303	/**
360	304	* free_tx_desc - reclaims Tx descriptors and their buffers
361		- * @adapter: the adapter
	305	+ * @adap: the adapter
362	306	* @q: the Tx queue to reclaim descriptors from
363	307	* @n: the number of descriptors to reclaim
364	308	* @unmap: whether the buffers should be unmapped for DMA
..	..	@@ -369,15 +313,16 @@
369	313	void free_tx_desc(struct adapter adap, struct sge_txq q,
370	314	unsigned int n, bool unmap)
371	315	{
372		- struct tx_sw_desc *d;
373	316	unsigned int cidx = q->cidx;
374		- struct device *dev = adap->pdev_dev;
	317	+ struct tx_sw_desc *d;
375	318
376	319	d = &q->sdesc[cidx];
377	320	while (n--) {
378	321	if (d->skb) { /* an SGL is present */
379		- if (unmap)
380		- unmap_sgl(dev, d->skb, d->sgl, q);
	322	+ if (unmap && d->addr[0]) {
	323	+ unmap_skb(adap->pdev_dev, d->skb, d->addr);
	324	+ memset(d->addr, 0, sizeof(d->addr));
	325	+ }
381	326	dev_consume_skb_any(d->skb);
382	327	d->skb = NULL;
383	328	}
..	..	@@ -401,6 +346,39 @@
401	346	}
402	347
403	348	/**
	349	+ * reclaim_completed_tx - reclaims completed TX Descriptors
	350	+ * @adap: the adapter
	351	+ * @q: the Tx queue to reclaim completed descriptors from
	352	+ * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1
	353	+ * @unmap: whether the buffers should be unmapped for DMA
	354	+ *
	355	+ * Reclaims Tx Descriptors that the SGE has indicated it has processed,
	356	+ * and frees the associated buffers if possible. If @max == -1, then
	357	+ * we'll use a defaiult maximum. Called with the TX Queue locked.
	358	+ */
	359	+static inline int reclaim_completed_tx(struct adapter adap, struct sge_txq q,
	360	+ int maxreclaim, bool unmap)
	361	+{
	362	+ int reclaim = reclaimable(q);
	363	+
	364	+ if (reclaim) {
	365	+ /*
	366	+ * Limit the amount of clean up work we do at a time to keep
	367	+ * the Tx lock hold time O(1).
	368	+ */
	369	+ if (maxreclaim < 0)
	370	+ maxreclaim = MAX_TX_RECLAIM;
	371	+ if (reclaim > maxreclaim)
	372	+ reclaim = maxreclaim;
	373	+
	374	+ free_tx_desc(adap, q, reclaim, unmap);
	375	+ q->in_use -= reclaim;
	376	+ }
	377	+
	378	+ return reclaim;
	379	+}
	380	+
	381	+/**
404	382	* cxgb4_reclaim_completed_tx - reclaims completed Tx descriptors
405	383	* @adap: the adapter
406	384	* @q: the Tx queue to reclaim completed descriptors from
..	..	@@ -410,22 +388,10 @@
410	388	* and frees the associated buffers if possible. Called with the Tx
411	389	* queue locked.
412	390	*/
413		-inline void cxgb4_reclaim_completed_tx(struct adapter adap, struct sge_txq q,
414		- bool unmap)
	391	+void cxgb4_reclaim_completed_tx(struct adapter adap, struct sge_txq q,
	392	+ bool unmap)
415	393	{
416		- int avail = reclaimable(q);
417		-
418		- if (avail) {
419		- /*
420		- * Limit the amount of clean up work we do at a time to keep
421		- * the Tx lock hold time O(1).
422		- */
423		- if (avail > MAX_TX_RECLAIM)
424		- avail = MAX_TX_RECLAIM;
425		-
426		- free_tx_desc(adap, q, avail, unmap);
427		- q->in_use -= avail;
428		- }
	394	+ (void)reclaim_completed_tx(adap, q, -1, unmap);
429	395	}
430	396	EXPORT_SYMBOL(cxgb4_reclaim_completed_tx);
431	397
..	..	@@ -454,7 +420,7 @@
454	420	break;
455	421
456	422	default:
457		- BUG_ON(1);
	423	+ BUG();
458	424	}
459	425
460	426	return buf_size;
..	..	@@ -694,7 +660,7 @@
694	660	{
695	661	size_t len = nelem * elem_size + stat_size;
696	662	void *s = NULL;
697		- void *p = dma_zalloc_coherent(dev, len, phys, GFP_KERNEL);
	663	+ void *p = dma_alloc_coherent(dev, len, phys, GFP_KERNEL);
698	664
699	665	if (!p)
700	666	return NULL;
..	..	@@ -756,6 +722,7 @@
756	722	/**
757	723	* is_eth_imm - can an Ethernet packet be sent as immediate data?
758	724	* @skb: the packet
	725	+ * @chip_ver: chip version
759	726	*
760	727	* Returns whether an Ethernet packet is small enough to fit as
761	728	* immediate data. Return value corresponds to headroom required.
..	..	@@ -768,6 +735,8 @@
768	735	chip_ver > CHELSIO_T5) {
769	736	hdrlen = sizeof(struct cpl_tx_tnl_lso);
770	737	hdrlen += sizeof(struct cpl_tx_pkt_core);
	738	+ } else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
	739	+ return 0;
771	740	} else {
772	741	hdrlen = skb_shinfo(skb)->gso_size ?
773	742	sizeof(struct cpl_tx_pkt_lso_core) : 0;
..	..	@@ -781,6 +750,7 @@
781	750	/**
782	751	* calc_tx_flits - calculate the number of flits for a packet Tx WR
783	752	* @skb: the packet
	753	+ * @chip_ver: chip version
784	754	*
785	755	* Returns the number of flits needed for a Tx WR for the given Ethernet
786	756	* packet, including the needed WR and CPL headers.
..	..	@@ -809,12 +779,20 @@
809	779	*/
810	780	flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
811	781	if (skb_shinfo(skb)->gso_size) {
812		- if (skb->encapsulation && chip_ver > CHELSIO_T5)
	782	+ if (skb->encapsulation && chip_ver > CHELSIO_T5) {
813	783	hdrlen = sizeof(struct fw_eth_tx_pkt_wr) +
814	784	sizeof(struct cpl_tx_tnl_lso);
815		- else
	785	+ } else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
	786	+ u32 pkt_hdrlen;
	787	+
	788	+ pkt_hdrlen = eth_get_headlen(skb->dev, skb->data,
	789	+ skb_headlen(skb));
	790	+ hdrlen = sizeof(struct fw_eth_tx_eo_wr) +
	791	+ round_up(pkt_hdrlen, 16);
	792	+ } else {
816	793	hdrlen = sizeof(struct fw_eth_tx_pkt_wr) +
817	794	sizeof(struct cpl_tx_pkt_lso_core);
	795	+ }
818	796
819	797	hdrlen += sizeof(struct cpl_tx_pkt_core);
820	798	flits += (hdrlen / sizeof(__be64));
..	..	@@ -828,6 +806,7 @@
828	806	/**
829	807	* calc_tx_descs - calculate the number of Tx descriptors for a packet
830	808	* @skb: the packet
	809	+ * @chip_ver: chip version
831	810	*
832	811	* Returns the number of Tx descriptors needed for the given Ethernet
833	812	* packet, including the needed WR and CPL headers.
..	..	@@ -910,6 +889,114 @@
910	889	*end = 0;
911	890	}
912	891	EXPORT_SYMBOL(cxgb4_write_sgl);
	892	+
	893	+/* cxgb4_write_partial_sgl - populate SGL for partial packet
	894	+ * @skb: the packet
	895	+ * @q: the Tx queue we are writing into
	896	+ * @sgl: starting location for writing the SGL
	897	+ * @end: points right after the end of the SGL
	898	+ * @addr: the list of bus addresses for the SGL elements
	899	+ * @start: start offset in the SKB where partial data starts
	900	+ * @len: length of data from @start to send out
	901	+ *
	902	+ * This API will handle sending out partial data of a skb if required.
	903	+ * Unlike cxgb4_write_sgl, @start can be any offset into the skb data,
	904	+ * and @len will decide how much data after @start offset to send out.
	905	+ */
	906	+void cxgb4_write_partial_sgl(const struct sk_buff skb, struct sge_txq q,
	907	+ struct ulptx_sgl sgl, u64 end,
	908	+ const dma_addr_t *addr, u32 start, u32 len)
	909	+{
	910	+ struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1] = {0}, *to;
	911	+ u32 frag_size, skb_linear_data_len = skb_headlen(skb);
	912	+ struct skb_shared_info *si = skb_shinfo(skb);
	913	+ u8 i = 0, frag_idx = 0, nfrags = 0;
	914	+ skb_frag_t *frag;
	915	+
	916	+ /* Fill the first SGL either from linear data or from partial
	917	+ * frag based on @start.
	918	+ */
	919	+ if (unlikely(start < skb_linear_data_len)) {
	920	+ frag_size = min(len, skb_linear_data_len - start);
	921	+ sgl->len0 = htonl(frag_size);
	922	+ sgl->addr0 = cpu_to_be64(addr[0] + start);
	923	+ len -= frag_size;
	924	+ nfrags++;
	925	+ } else {
	926	+ start -= skb_linear_data_len;
	927	+ frag = &si->frags[frag_idx];
	928	+ frag_size = skb_frag_size(frag);
	929	+ /* find the first frag */
	930	+ while (start >= frag_size) {
	931	+ start -= frag_size;
	932	+ frag_idx++;
	933	+ frag = &si->frags[frag_idx];
	934	+ frag_size = skb_frag_size(frag);
	935	+ }
	936	+
	937	+ frag_size = min(len, skb_frag_size(frag) - start);
	938	+ sgl->len0 = cpu_to_be32(frag_size);
	939	+ sgl->addr0 = cpu_to_be64(addr[frag_idx + 1] + start);
	940	+ len -= frag_size;
	941	+ nfrags++;
	942	+ frag_idx++;
	943	+ }
	944	+
	945	+ /* If the entire partial data fit in one SGL, then send it out
	946	+ * now.
	947	+ */
	948	+ if (!len)
	949	+ goto done;
	950	+
	951	+ /* Most of the complexity below deals with the possibility we hit the
	952	+ * end of the queue in the middle of writing the SGL. For this case
	953	+ * only we create the SGL in a temporary buffer and then copy it.
	954	+ */
	955	+ to = (u8 )end > (u8 )q->stat ? buf : sgl->sge;
	956	+
	957	+ /* If the skb couldn't fit in first SGL completely, fill the
	958	+ * rest of the frags in subsequent SGLs. Note that each SGL
	959	+ * pair can store 2 frags.
	960	+ */
	961	+ while (len) {
	962	+ frag_size = min(len, skb_frag_size(&si->frags[frag_idx]));
	963	+ to->len[i & 1] = cpu_to_be32(frag_size);
	964	+ to->addr[i & 1] = cpu_to_be64(addr[frag_idx + 1]);
	965	+ if (i && (i & 1))
	966	+ to++;
	967	+ nfrags++;
	968	+ frag_idx++;
	969	+ i++;
	970	+ len -= frag_size;
	971	+ }
	972	+
	973	+ /* If we ended in an odd boundary, then set the second SGL's
	974	+ * length in the pair to 0.
	975	+ */
	976	+ if (i & 1)
	977	+ to->len[1] = cpu_to_be32(0);
	978	+
	979	+ /* Copy from temporary buffer to Tx ring, in case we hit the
	980	+ * end of the queue in the middle of writing the SGL.
	981	+ */
	982	+ if (unlikely((u8 )end > (u8 )q->stat)) {
	983	+ u32 part0 = (u8 )q->stat - (u8 )sgl->sge, part1;
	984	+
	985	+ if (likely(part0))
	986	+ memcpy(sgl->sge, buf, part0);
	987	+ part1 = (u8 )end - (u8 )q->stat;
	988	+ memcpy(q->desc, (u8 *)buf + part0, part1);
	989	+ end = (void *)q->desc + part1;
	990	+ }
	991	+
	992	+ /* 0-pad to multiple of 16 */
	993	+ if ((uintptr_t)end & 8)
	994	+ *end = 0;
	995	+done:
	996	+ sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) \|
	997	+ ULPTX_NSGE_V(nfrags));
	998	+}
	999	+EXPORT_SYMBOL(cxgb4_write_partial_sgl);
913	1000
914	1001	/* This function copies 64 byte coalesced work request to
915	1002	* memory mapped BAR2 space. For coalesced WR SGE fetches
..	..	@@ -1287,6 +1374,123 @@
1287	1374	tnl_lso->EthLenOffset_Size = htonl(CPL_TX_TNL_LSO_SIZE_V(skb->len));
1288	1375	}
1289	1376
	1377	+static inline void write_tso_wr(struct adapter adap, struct sk_buff *skb,
	1378	+ struct cpl_tx_pkt_lso_core *lso)
	1379	+{
	1380	+ int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
	1381	+ int l3hdr_len = skb_network_header_len(skb);
	1382	+ const struct skb_shared_info *ssi;
	1383	+ bool ipv6 = false;
	1384	+
	1385	+ ssi = skb_shinfo(skb);
	1386	+ if (ssi->gso_type & SKB_GSO_TCPV6)
	1387	+ ipv6 = true;
	1388	+
	1389	+ lso->lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) \|
	1390	+ LSO_FIRST_SLICE_F \| LSO_LAST_SLICE_F \|
	1391	+ LSO_IPV6_V(ipv6) \|
	1392	+ LSO_ETHHDR_LEN_V(eth_xtra_len / 4) \|
	1393	+ LSO_IPHDR_LEN_V(l3hdr_len / 4) \|
	1394	+ LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
	1395	+ lso->ipid_ofst = htons(0);
	1396	+ lso->mss = htons(ssi->gso_size);
	1397	+ lso->seqno_offset = htonl(0);
	1398	+ if (is_t4(adap->params.chip))
	1399	+ lso->len = htonl(skb->len);
	1400	+ else
	1401	+ lso->len = htonl(LSO_T5_XFER_SIZE_V(skb->len));
	1402	+
	1403	+ return (void *)(lso + 1);
	1404	+}
	1405	+
	1406	+/**
	1407	+ * t4_sge_eth_txq_egress_update - handle Ethernet TX Queue update
	1408	+ * @adap: the adapter
	1409	+ * @eq: the Ethernet TX Queue
	1410	+ * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1
	1411	+ *
	1412	+ * We're typically called here to update the state of an Ethernet TX
	1413	+ * Queue with respect to the hardware's progress in consuming the TX
	1414	+ * Work Requests that we've put on that Egress Queue. This happens
	1415	+ * when we get Egress Queue Update messages and also prophylactically
	1416	+ * in regular timer-based Ethernet TX Queue maintenance.
	1417	+ */
	1418	+int t4_sge_eth_txq_egress_update(struct adapter adap, struct sge_eth_txq eq,
	1419	+ int maxreclaim)
	1420	+{
	1421	+ unsigned int reclaimed, hw_cidx;
	1422	+ struct sge_txq *q = &eq->q;
	1423	+ int hw_in_use;
	1424	+
	1425	+ if (!q->in_use \|\| !__netif_tx_trylock(eq->txq))
	1426	+ return 0;
	1427	+
	1428	+ /* Reclaim pending completed TX Descriptors. */
	1429	+ reclaimed = reclaim_completed_tx(adap, &eq->q, maxreclaim, true);
	1430	+
	1431	+ hw_cidx = ntohs(READ_ONCE(q->stat->cidx));
	1432	+ hw_in_use = q->pidx - hw_cidx;
	1433	+ if (hw_in_use < 0)
	1434	+ hw_in_use += q->size;
	1435	+
	1436	+ /* If the TX Queue is currently stopped and there's now more than half
	1437	+ * the queue available, restart it. Otherwise bail out since the rest
	1438	+ * of what we want do here is with the possibility of shipping any
	1439	+ * currently buffered Coalesced TX Work Request.
	1440	+ */
	1441	+ if (netif_tx_queue_stopped(eq->txq) && hw_in_use < (q->size / 2)) {
	1442	+ netif_tx_wake_queue(eq->txq);
	1443	+ eq->q.restarts++;
	1444	+ }
	1445	+
	1446	+ __netif_tx_unlock(eq->txq);
	1447	+ return reclaimed;
	1448	+}
	1449	+
	1450	+static inline int cxgb4_validate_skb(struct sk_buff *skb,
	1451	+ struct net_device *dev,
	1452	+ u32 min_pkt_len)
	1453	+{
	1454	+ u32 max_pkt_len;
	1455	+
	1456	+ /* The chip min packet length is 10 octets but some firmware
	1457	+ * commands have a minimum packet length requirement. So, play
	1458	+ * safe and reject anything shorter than @min_pkt_len.
	1459	+ */
	1460	+ if (unlikely(skb->len < min_pkt_len))
	1461	+ return -EINVAL;
	1462	+
	1463	+ /* Discard the packet if the length is greater than mtu */
	1464	+ max_pkt_len = ETH_HLEN + dev->mtu;
	1465	+
	1466	+ if (skb_vlan_tagged(skb))
	1467	+ max_pkt_len += VLAN_HLEN;
	1468	+
	1469	+ if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
	1470	+ return -EINVAL;
	1471	+
	1472	+ return 0;
	1473	+}
	1474	+
	1475	+static void write_eo_udp_wr(struct sk_buff skb, struct fw_eth_tx_eo_wr *wr,
	1476	+ u32 hdr_len)
	1477	+{
	1478	+ wr->u.udpseg.type = FW_ETH_TX_EO_TYPE_UDPSEG;
	1479	+ wr->u.udpseg.ethlen = skb_network_offset(skb);
	1480	+ wr->u.udpseg.iplen = cpu_to_be16(skb_network_header_len(skb));
	1481	+ wr->u.udpseg.udplen = sizeof(struct udphdr);
	1482	+ wr->u.udpseg.rtplen = 0;
	1483	+ wr->u.udpseg.r4 = 0;
	1484	+ if (skb_shinfo(skb)->gso_size)
	1485	+ wr->u.udpseg.mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
	1486	+ else
	1487	+ wr->u.udpseg.mss = cpu_to_be16(skb->len - hdr_len);
	1488	+ wr->u.udpseg.schedpktsize = wr->u.udpseg.mss;
	1489	+ wr->u.udpseg.plen = cpu_to_be32(skb->len - hdr_len);
	1490	+
	1491	+ return (void *)(wr + 1);
	1492	+}
	1493	+
1290	1494	/**
1291	1495	* cxgb4_eth_xmit - add a packet to an Ethernet Tx queue
1292	1496	* @skb: the packet
..	..	@@ -1296,59 +1500,47 @@
1296	1500	*/
1297	1501	static netdev_tx_t cxgb4_eth_xmit(struct sk_buff skb, struct net_device dev)
1298	1502	{
1299		- u32 wr_mid, ctrl0, op;
1300		- u64 cntrl, end, sgl;
1301		- int qidx, credits;
1302		- unsigned int flits, ndesc;
1303		- struct adapter *adap;
1304		- struct sge_eth_txq *q;
1305		- const struct port_info *pi;
	1503	+ enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE;
	1504	+ bool ptp_enabled = is_ptp_enabled(skb, dev);
	1505	+ unsigned int last_desc, flits, ndesc;
	1506	+ u32 wr_mid, ctrl0, op, sgl_off = 0;
	1507	+ const struct skb_shared_info *ssi;
	1508	+ int len, qidx, credits, ret, left;
	1509	+ struct tx_sw_desc *sgl_sdesc;
	1510	+ struct fw_eth_tx_eo_wr *eowr;
1306	1511	struct fw_eth_tx_pkt_wr *wr;
1307	1512	struct cpl_tx_pkt_core *cpl;
1308		- const struct skb_shared_info *ssi;
1309		- dma_addr_t addr[MAX_SKB_FRAGS + 1];
	1513	+ const struct port_info *pi;
1310	1514	bool immediate = false;
1311		- int len, max_pkt_len;
1312		- bool ptp_enabled = is_ptp_enabled(skb, dev);
	1515	+ u64 cntrl, end, sgl;
	1516	+ struct sge_eth_txq *q;
1313	1517	unsigned int chip_ver;
1314		- enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE;
	1518	+ struct adapter *adap;
1315	1519
1316		-#ifdef CONFIG_CHELSIO_T4_FCOE
1317		- int err;
1318		-#endif /* CONFIG_CHELSIO_T4_FCOE */
1319		-
1320		- /*
1321		- * The chip min packet length is 10 octets but play safe and reject
1322		- * anything shorter than an Ethernet header.
1323		- */
1324		- if (unlikely(skb->len < ETH_HLEN)) {
1325		-out_free: dev_kfree_skb_any(skb);
1326		- return NETDEV_TX_OK;
1327		- }
1328		-
1329		- /* Discard the packet if the length is greater than mtu */
1330		- max_pkt_len = ETH_HLEN + dev->mtu;
1331		- if (skb_vlan_tagged(skb))
1332		- max_pkt_len += VLAN_HLEN;
1333		- if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
	1520	+ ret = cxgb4_validate_skb(skb, dev, ETH_HLEN);
	1521	+ if (ret)
1334	1522	goto out_free;
1335	1523
1336	1524	pi = netdev_priv(dev);
1337	1525	adap = pi->adapter;
1338	1526	ssi = skb_shinfo(skb);
1339		-#ifdef CONFIG_CHELSIO_IPSEC_INLINE
	1527	+#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE)
1340	1528	if (xfrm_offload(skb) && !ssi->gso_size)
1341		- return adap->uld[CXGB4_ULD_CRYPTO].tx_handler(skb, dev);
	1529	+ return adap->uld[CXGB4_ULD_IPSEC].tx_handler(skb, dev);
1342	1530	#endif /* CHELSIO_IPSEC_INLINE */
	1531	+
	1532	+#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE)
	1533	+ if (cxgb4_is_ktls_skb(skb) &&
	1534	+ (skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb))))
	1535	+ return adap->uld[CXGB4_ULD_KTLS].tx_handler(skb, dev);
	1536	+#endif /* CHELSIO_TLS_DEVICE */
1343	1537
1344	1538	qidx = skb_get_queue_mapping(skb);
1345	1539	if (ptp_enabled) {
1346		- spin_lock(&adap->ptp_lock);
1347	1540	if (!(adap->ptp_tx_skb)) {
1348	1541	skb_shinfo(skb)->tx_flags \|= SKBTX_IN_PROGRESS;
1349	1542	adap->ptp_tx_skb = skb_get(skb);
1350	1543	} else {
1351		- spin_unlock(&adap->ptp_lock);
1352	1544	goto out_free;
1353	1545	}
1354	1546	q = &adap->sge.ptptxq;
..	..	@@ -1357,16 +1549,13 @@
1357	1549	}
1358	1550	skb_tx_timestamp(skb);
1359	1551
1360		- cxgb4_reclaim_completed_tx(adap, &q->q, true);
	1552	+ reclaim_completed_tx(adap, &q->q, -1, true);
1361	1553	cntrl = TXPKT_L4CSUM_DIS_F \| TXPKT_IPCSUM_DIS_F;
1362	1554
1363	1555	#ifdef CONFIG_CHELSIO_T4_FCOE
1364		- err = cxgb_fcoe_offload(skb, adap, pi, &cntrl);
1365		- if (unlikely(err == -ENOTSUPP)) {
1366		- if (ptp_enabled)
1367		- spin_unlock(&adap->ptp_lock);
	1556	+ ret = cxgb_fcoe_offload(skb, adap, pi, &cntrl);
	1557	+ if (unlikely(ret == -EOPNOTSUPP))
1368	1558	goto out_free;
1369		- }
1370	1559	#endif /* CONFIG_CHELSIO_T4_FCOE */
1371	1560
1372	1561	chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
..	..	@@ -1379,8 +1568,6 @@
1379	1568	dev_err(adap->pdev_dev,
1380	1569	"%s: Tx ring %u full while queue awake!\n",
1381	1570	dev->name, qidx);
1382		- if (ptp_enabled)
1383		- spin_unlock(&adap->ptp_lock);
1384	1571	return NETDEV_TX_BUSY;
1385	1572	}
1386	1573
..	..	@@ -1390,32 +1577,45 @@
1390	1577	if (skb->encapsulation && chip_ver > CHELSIO_T5)
1391	1578	tnl_type = cxgb_encap_offload_supported(skb);
1392	1579
	1580	+ last_desc = q->q.pidx + ndesc - 1;
	1581	+ if (last_desc >= q->q.size)
	1582	+ last_desc -= q->q.size;
	1583	+ sgl_sdesc = &q->q.sdesc[last_desc];
	1584	+
1393	1585	if (!immediate &&
1394		- unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) {
	1586	+ unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
	1587	+ memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1395	1588	q->mapping_err++;
1396		- if (ptp_enabled)
1397		- spin_unlock(&adap->ptp_lock);
1398	1589	goto out_free;
1399	1590	}
1400	1591
1401	1592	wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
1402	1593	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
	1594	+ /* After we're done injecting the Work Request for this
	1595	+ * packet, we'll be below our "stop threshold" so stop the TX
	1596	+ * Queue now and schedule a request for an SGE Egress Queue
	1597	+ * Update message. The queue will get started later on when
	1598	+ * the firmware processes this Work Request and sends us an
	1599	+ * Egress Queue Status Update message indicating that space
	1600	+ * has opened up.
	1601	+ */
1403	1602	eth_txq_stop(q);
1404	1603	wr_mid \|= FW_WR_EQUEQ_F \| FW_WR_EQUIQ_F;
1405	1604	}
1406	1605
1407	1606	wr = (void *)&q->q.desc[q->q.pidx];
	1607	+ eowr = (void *)&q->q.desc[q->q.pidx];
1408	1608	wr->equiq_to_len16 = htonl(wr_mid);
1409	1609	wr->r3 = cpu_to_be64(0);
1410		- end = (u64 *)wr + flits;
	1610	+ if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
	1611	+ end = (u64 *)eowr + flits;
	1612	+ else
	1613	+ end = (u64 *)wr + flits;
1411	1614
1412	1615	len = immediate ? skb->len : 0;
1413	1616	len += sizeof(*cpl);
1414		- if (ssi->gso_size) {
	1617	+ if (ssi->gso_size && !(ssi->gso_type & SKB_GSO_UDP_L4)) {
1415	1618	struct cpl_tx_pkt_lso_core lso = (void )(wr + 1);
1416		- bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
1417		- int l3hdr_len = skb_network_header_len(skb);
1418		- int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1419	1619	struct cpl_tx_tnl_lso tnl_lso = (void )(wr + 1);
1420	1620
1421	1621	if (tnl_type)
..	..	@@ -1436,51 +1636,37 @@
1436	1636	if (iph->version == 4) {
1437	1637	iph->check = 0;
1438	1638	iph->tot_len = 0;
1439		- iph->check = (u16)(~ip_fast_csum((u8 *)iph,
1440		- iph->ihl));
	1639	+ iph->check = ~ip_fast_csum((u8 *)iph, iph->ihl);
1441	1640	}
1442	1641	if (skb->ip_summed == CHECKSUM_PARTIAL)
1443	1642	cntrl = hwcsum(adap->params.chip, skb);
1444	1643	} else {
1445		- lso->lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) \|
1446		- LSO_FIRST_SLICE_F \| LSO_LAST_SLICE_F \|
1447		- LSO_IPV6_V(v6) \|
1448		- LSO_ETHHDR_LEN_V(eth_xtra_len / 4) \|
1449		- LSO_IPHDR_LEN_V(l3hdr_len / 4) \|
1450		- LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
1451		- lso->ipid_ofst = htons(0);
1452		- lso->mss = htons(ssi->gso_size);
1453		- lso->seqno_offset = htonl(0);
1454		- if (is_t4(adap->params.chip))
1455		- lso->len = htonl(skb->len);
1456		- else
1457		- lso->len = htonl(LSO_T5_XFER_SIZE_V(skb->len));
1458		- cpl = (void *)(lso + 1);
1459		-
1460		- if (CHELSIO_CHIP_VERSION(adap->params.chip)
1461		- <= CHELSIO_T5)
1462		- cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1463		- else
1464		- cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1465		-
1466		- cntrl \|= TXPKT_CSUM_TYPE_V(v6 ?
1467		- TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) \|
1468		- TXPKT_IPHDR_LEN_V(l3hdr_len);
	1644	+ cpl = write_tso_wr(adap, skb, lso);
	1645	+ cntrl = hwcsum(adap->params.chip, skb);
1469	1646	}
1470	1647	sgl = (u64 )(cpl + 1); / sgl start here */
1471		- if (unlikely((u8 )sgl >= (u8 )q->q.stat)) {
1472		- /* If current position is already at the end of the
1473		- * txq, reset the current to point to start of the queue
1474		- * and update the end ptr as well.
1475		- */
1476		- if (sgl == (u64 *)q->q.stat) {
1477		- int left = (u8 )end - (u8 )q->q.stat;
1478		-
1479		- end = (void *)q->q.desc + left;
1480		- sgl = (void *)q->q.desc;
1481		- }
1482		- }
1483	1648	q->tso++;
	1649	+ q->tx_cso += ssi->gso_segs;
	1650	+ } else if (ssi->gso_size) {
	1651	+ u64 *start;
	1652	+ u32 hdrlen;
	1653	+
	1654	+ hdrlen = eth_get_headlen(dev, skb->data, skb_headlen(skb));
	1655	+ len += hdrlen;
	1656	+ wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_EO_WR) \|
	1657	+ FW_ETH_TX_EO_WR_IMMDLEN_V(len));
	1658	+ cpl = write_eo_udp_wr(skb, eowr, hdrlen);
	1659	+ cntrl = hwcsum(adap->params.chip, skb);
	1660	+
	1661	+ start = (u64 *)(cpl + 1);
	1662	+ sgl = (u64 )inline_tx_skb_header(skb, &q->q, (void )start,
	1663	+ hdrlen);
	1664	+ if (unlikely(start > sgl)) {
	1665	+ left = (u8 )end - (u8 )q->q.stat;
	1666	+ end = (void *)q->q.desc + left;
	1667	+ }
	1668	+ sgl_off = hdrlen;
	1669	+ q->uso++;
1484	1670	q->tx_cso += ssi->gso_segs;
1485	1671	} else {
1486	1672	if (ptp_enabled)
..	..	@@ -1496,6 +1682,16 @@
1496	1682	TXPKT_IPCSUM_DIS_F;
1497	1683	q->tx_cso++;
1498	1684	}
	1685	+ }
	1686	+
	1687	+ if (unlikely((u8 )sgl >= (u8 )q->q.stat)) {
	1688	+ /* If current position is already at the end of the
	1689	+ * txq, reset the current to point to start of the queue
	1690	+ * and update the end ptr as well.
	1691	+ */
	1692	+ left = (u8 )end - (u8 )q->q.stat;
	1693	+ end = (void *)q->q.desc + left;
	1694	+ sgl = (void *)q->q.desc;
1499	1695	}
1500	1696
1501	1697	if (skb_vlan_tag_present(skb)) {
..	..	@@ -1527,23 +1723,19 @@
1527	1723	cxgb4_inline_tx_skb(skb, &q->q, sgl);
1528	1724	dev_consume_skb_any(skb);
1529	1725	} else {
1530		- int last_desc;
1531		-
1532		- cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, 0, addr);
	1726	+ cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, sgl_off,
	1727	+ sgl_sdesc->addr);
1533	1728	skb_orphan(skb);
1534		-
1535		- last_desc = q->q.pidx + ndesc - 1;
1536		- if (last_desc >= q->q.size)
1537		- last_desc -= q->q.size;
1538		- q->q.sdesc[last_desc].skb = skb;
1539		- q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)sgl;
	1729	+ sgl_sdesc->skb = skb;
1540	1730	}
1541	1731
1542	1732	txq_advance(&q->q, ndesc);
1543	1733
1544	1734	cxgb4_ring_tx_db(adap, &q->q, ndesc);
1545		- if (ptp_enabled)
1546		- spin_unlock(&adap->ptp_lock);
	1735	+ return NETDEV_TX_OK;
	1736	+
	1737	+out_free:
	1738	+ dev_kfree_skb_any(skb);
1547	1739	return NETDEV_TX_OK;
1548	1740	}
1549	1741
..	..	@@ -1630,35 +1822,28 @@
1630	1822	static netdev_tx_t cxgb4_vf_eth_xmit(struct sk_buff *skb,
1631	1823	struct net_device *dev)
1632	1824	{
1633		- dma_addr_t addr[MAX_SKB_FRAGS + 1];
	1825	+ unsigned int last_desc, flits, ndesc;
1634	1826	const struct skb_shared_info *ssi;
1635	1827	struct fw_eth_tx_pkt_vm_wr *wr;
1636		- int qidx, credits, max_pkt_len;
	1828	+ struct tx_sw_desc *sgl_sdesc;
1637	1829	struct cpl_tx_pkt_core *cpl;
1638	1830	const struct port_info *pi;
1639		- unsigned int flits, ndesc;
1640	1831	struct sge_eth_txq *txq;
1641	1832	struct adapter *adapter;
	1833	+ int qidx, credits, ret;
	1834	+ size_t fw_hdr_copy_len;
1642	1835	u64 cntrl, *end;
1643	1836	u32 wr_mid;
1644		- const size_t fw_hdr_copy_len = sizeof(wr->ethmacdst) +
1645		- sizeof(wr->ethmacsrc) +
1646		- sizeof(wr->ethtype) +
1647		- sizeof(wr->vlantci);
1648	1837
1649	1838	/* The chip minimum packet length is 10 octets but the firmware
1650	1839	* command that we are using requires that we copy the Ethernet header
1651	1840	* (including the VLAN tag) into the header so we reject anything
1652	1841	* smaller than that ...
1653	1842	*/
1654		- if (unlikely(skb->len < fw_hdr_copy_len))
1655		- goto out_free;
1656		-
1657		- /* Discard the packet if the length is greater than mtu */
1658		- max_pkt_len = ETH_HLEN + dev->mtu;
1659		- if (skb_vlan_tag_present(skb))
1660		- max_pkt_len += VLAN_HLEN;
1661		- if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
	1843	+ fw_hdr_copy_len = sizeof(wr->ethmacdst) + sizeof(wr->ethmacsrc) +
	1844	+ sizeof(wr->ethtype) + sizeof(wr->vlantci);
	1845	+ ret = cxgb4_validate_skb(skb, dev, fw_hdr_copy_len);
	1846	+ if (ret)
1662	1847	goto out_free;
1663	1848
1664	1849	/* Figure out which TX Queue we're going to use. */
..	..	@@ -1671,7 +1856,7 @@
1671	1856	/* Take this opportunity to reclaim any TX Descriptors whose DMA
1672	1857	* transfers have completed.
1673	1858	*/
1674		- cxgb4_reclaim_completed_tx(adapter, &txq->q, true);
	1859	+ reclaim_completed_tx(adapter, &txq->q, -1, true);
1675	1860
1676	1861	/* Calculate the number of flits and TX Descriptors we're going to
1677	1862	* need along with how many TX Descriptors will be left over after
..	..	@@ -1694,12 +1879,19 @@
1694	1879	return NETDEV_TX_BUSY;
1695	1880	}
1696	1881
	1882	+ last_desc = txq->q.pidx + ndesc - 1;
	1883	+ if (last_desc >= txq->q.size)
	1884	+ last_desc -= txq->q.size;
	1885	+ sgl_sdesc = &txq->q.sdesc[last_desc];
	1886	+
1697	1887	if (!t4vf_is_eth_imm(skb) &&
1698		- unlikely(cxgb4_map_skb(adapter->pdev_dev, skb, addr) < 0)) {
	1888	+ unlikely(cxgb4_map_skb(adapter->pdev_dev, skb,
	1889	+ sgl_sdesc->addr) < 0)) {
1699	1890	/* We need to map the skb into PCI DMA space (because it can't
1700	1891	* be in-lined directly into the Work Request) and the mapping
1701	1892	* operation failed. Record the error and drop the packet.
1702	1893	*/
	1894	+ memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1703	1895	txq->mapping_err++;
1704	1896	goto out_free;
1705	1897	}
..	..	@@ -1865,7 +2057,6 @@
1865	2057	*/
1866	2058	struct ulptx_sgl sgl = (struct ulptx_sgl )(cpl + 1);
1867	2059	struct sge_txq *tq = &txq->q;
1868		- int last_desc;
1869	2060
1870	2061	/* If the Work Request header was an exact multiple of our TX
1871	2062	* Descriptor length, then it's possible that the starting SGL
..	..	@@ -1879,14 +2070,9 @@
1879	2070	((void )end - (void )tq->stat));
1880	2071	}
1881	2072
1882		- cxgb4_write_sgl(skb, tq, sgl, end, 0, addr);
	2073	+ cxgb4_write_sgl(skb, tq, sgl, end, 0, sgl_sdesc->addr);
1883	2074	skb_orphan(skb);
1884		-
1885		- last_desc = tq->pidx + ndesc - 1;
1886		- if (last_desc >= tq->size)
1887		- last_desc -= tq->size;
1888		- tq->sdesc[last_desc].skb = skb;
1889		- tq->sdesc[last_desc].sgl = sgl;
	2075	+ sgl_sdesc->skb = skb;
1890	2076	}
1891	2077
1892	2078	/* Advance our internal TX Queue state, tell the hardware about
..	..	@@ -1905,23 +2091,13 @@
1905	2091	return NETDEV_TX_OK;
1906	2092	}
1907	2093
1908		-netdev_tx_t t4_start_xmit(struct sk_buff skb, struct net_device dev)
1909		-{
1910		- struct port_info *pi = netdev_priv(dev);
1911		-
1912		- if (unlikely(pi->eth_flags & PRIV_FLAG_PORT_TX_VM))
1913		- return cxgb4_vf_eth_xmit(skb, dev);
1914		-
1915		- return cxgb4_eth_xmit(skb, dev);
1916		-}
1917		-
1918	2094	/**
1919		- * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1920		- * @q: the SGE control Tx queue
	2095	+ * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
	2096	+ * @q: the SGE control Tx queue
1921	2097	*
1922		- * This is a variant of cxgb4_reclaim_completed_tx() that is used
1923		- * for Tx queues that send only immediate data (presently just
1924		- * the control queues) and thus do not have any sk_buffs to release.
	2098	+ * This is a variant of cxgb4_reclaim_completed_tx() that is used
	2099	+ * for Tx queues that send only immediate data (presently just
	2100	+ * the control queues) and thus do not have any sk_buffs to release.
1925	2101	*/
1926	2102	static inline void reclaim_completed_tx_imm(struct sge_txq *q)
1927	2103	{
..	..	@@ -1933,6 +2109,518 @@
1933	2109
1934	2110	q->in_use -= reclaim;
1935	2111	q->cidx = hw_cidx;
	2112	+}
	2113	+
	2114	+static inline void eosw_txq_advance_index(u32 *idx, u32 n, u32 max)
	2115	+{
	2116	+ u32 val = *idx + n;
	2117	+
	2118	+ if (val >= max)
	2119	+ val -= max;
	2120	+
	2121	+ *idx = val;
	2122	+}
	2123	+
	2124	+void cxgb4_eosw_txq_free_desc(struct adapter *adap,
	2125	+ struct sge_eosw_txq *eosw_txq, u32 ndesc)
	2126	+{
	2127	+ struct tx_sw_desc *d;
	2128	+
	2129	+ d = &eosw_txq->desc[eosw_txq->last_cidx];
	2130	+ while (ndesc--) {
	2131	+ if (d->skb) {
	2132	+ if (d->addr[0]) {
	2133	+ unmap_skb(adap->pdev_dev, d->skb, d->addr);
	2134	+ memset(d->addr, 0, sizeof(d->addr));
	2135	+ }
	2136	+ dev_consume_skb_any(d->skb);
	2137	+ d->skb = NULL;
	2138	+ }
	2139	+ eosw_txq_advance_index(&eosw_txq->last_cidx, 1,
	2140	+ eosw_txq->ndesc);
	2141	+ d = &eosw_txq->desc[eosw_txq->last_cidx];
	2142	+ }
	2143	+}
	2144	+
	2145	+static inline void eosw_txq_advance(struct sge_eosw_txq *eosw_txq, u32 n)
	2146	+{
	2147	+ eosw_txq_advance_index(&eosw_txq->pidx, n, eosw_txq->ndesc);
	2148	+ eosw_txq->inuse += n;
	2149	+}
	2150	+
	2151	+static inline int eosw_txq_enqueue(struct sge_eosw_txq *eosw_txq,
	2152	+ struct sk_buff *skb)
	2153	+{
	2154	+ if (eosw_txq->inuse == eosw_txq->ndesc)
	2155	+ return -ENOMEM;
	2156	+
	2157	+ eosw_txq->desc[eosw_txq->pidx].skb = skb;
	2158	+ return 0;
	2159	+}
	2160	+
	2161	+static inline struct sk_buff eosw_txq_peek(struct sge_eosw_txq eosw_txq)
	2162	+{
	2163	+ return eosw_txq->desc[eosw_txq->last_pidx].skb;
	2164	+}
	2165	+
	2166	+static inline u8 ethofld_calc_tx_flits(struct adapter *adap,
	2167	+ struct sk_buff *skb, u32 hdr_len)
	2168	+{
	2169	+ u8 flits, nsgl = 0;
	2170	+ u32 wrlen;
	2171	+
	2172	+ wrlen = sizeof(struct fw_eth_tx_eo_wr) + sizeof(struct cpl_tx_pkt_core);
	2173	+ if (skb_shinfo(skb)->gso_size &&
	2174	+ !(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4))
	2175	+ wrlen += sizeof(struct cpl_tx_pkt_lso_core);
	2176	+
	2177	+ wrlen += roundup(hdr_len, 16);
	2178	+
	2179	+ /* Packet headers + WR + CPLs */
	2180	+ flits = DIV_ROUND_UP(wrlen, 8);
	2181	+
	2182	+ if (skb_shinfo(skb)->nr_frags > 0) {
	2183	+ if (skb_headlen(skb) - hdr_len)
	2184	+ nsgl = sgl_len(skb_shinfo(skb)->nr_frags + 1);
	2185	+ else
	2186	+ nsgl = sgl_len(skb_shinfo(skb)->nr_frags);
	2187	+ } else if (skb->len - hdr_len) {
	2188	+ nsgl = sgl_len(1);
	2189	+ }
	2190	+
	2191	+ return flits + nsgl;
	2192	+}
	2193	+
	2194	+static void write_eo_wr(struct adapter adap, struct sge_eosw_txq *eosw_txq,
	2195	+ struct sk_buff skb, struct fw_eth_tx_eo_wr wr,
	2196	+ u32 hdr_len, u32 wrlen)
	2197	+{
	2198	+ const struct skb_shared_info *ssi = skb_shinfo(skb);
	2199	+ struct cpl_tx_pkt_core *cpl;
	2200	+ u32 immd_len, wrlen16;
	2201	+ bool compl = false;
	2202	+ u8 ver, proto;
	2203	+
	2204	+ ver = ip_hdr(skb)->version;
	2205	+ proto = (ver == 6) ? ipv6_hdr(skb)->nexthdr : ip_hdr(skb)->protocol;
	2206	+
	2207	+ wrlen16 = DIV_ROUND_UP(wrlen, 16);
	2208	+ immd_len = sizeof(struct cpl_tx_pkt_core);
	2209	+ if (skb_shinfo(skb)->gso_size &&
	2210	+ !(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4))
	2211	+ immd_len += sizeof(struct cpl_tx_pkt_lso_core);
	2212	+ immd_len += hdr_len;
	2213	+
	2214	+ if (!eosw_txq->ncompl \|\|
	2215	+ (eosw_txq->last_compl + wrlen16) >=
	2216	+ (adap->params.ofldq_wr_cred / 2)) {
	2217	+ compl = true;
	2218	+ eosw_txq->ncompl++;
	2219	+ eosw_txq->last_compl = 0;
	2220	+ }
	2221	+
	2222	+ wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_EO_WR) \|
	2223	+ FW_ETH_TX_EO_WR_IMMDLEN_V(immd_len) \|
	2224	+ FW_WR_COMPL_V(compl));
	2225	+ wr->equiq_to_len16 = cpu_to_be32(FW_WR_LEN16_V(wrlen16) \|
	2226	+ FW_WR_FLOWID_V(eosw_txq->hwtid));
	2227	+ wr->r3 = 0;
	2228	+ if (proto == IPPROTO_UDP) {
	2229	+ cpl = write_eo_udp_wr(skb, wr, hdr_len);
	2230	+ } else {
	2231	+ wr->u.tcpseg.type = FW_ETH_TX_EO_TYPE_TCPSEG;
	2232	+ wr->u.tcpseg.ethlen = skb_network_offset(skb);
	2233	+ wr->u.tcpseg.iplen = cpu_to_be16(skb_network_header_len(skb));
	2234	+ wr->u.tcpseg.tcplen = tcp_hdrlen(skb);
	2235	+ wr->u.tcpseg.tsclk_tsoff = 0;
	2236	+ wr->u.tcpseg.r4 = 0;
	2237	+ wr->u.tcpseg.r5 = 0;
	2238	+ wr->u.tcpseg.plen = cpu_to_be32(skb->len - hdr_len);
	2239	+
	2240	+ if (ssi->gso_size) {
	2241	+ struct cpl_tx_pkt_lso_core lso = (void )(wr + 1);
	2242	+
	2243	+ wr->u.tcpseg.mss = cpu_to_be16(ssi->gso_size);
	2244	+ cpl = write_tso_wr(adap, skb, lso);
	2245	+ } else {
	2246	+ wr->u.tcpseg.mss = cpu_to_be16(0xffff);
	2247	+ cpl = (void *)(wr + 1);
	2248	+ }
	2249	+ }
	2250	+
	2251	+ eosw_txq->cred -= wrlen16;
	2252	+ eosw_txq->last_compl += wrlen16;
	2253	+ return cpl;
	2254	+}
	2255	+
	2256	+static int ethofld_hard_xmit(struct net_device *dev,
	2257	+ struct sge_eosw_txq *eosw_txq)
	2258	+{
	2259	+ struct port_info *pi = netdev2pinfo(dev);
	2260	+ struct adapter *adap = netdev2adap(dev);
	2261	+ u32 wrlen, wrlen16, hdr_len, data_len;
	2262	+ enum sge_eosw_state next_state;
	2263	+ u64 cntrl, start, end, *sgl;
	2264	+ struct sge_eohw_txq *eohw_txq;
	2265	+ struct cpl_tx_pkt_core *cpl;
	2266	+ struct fw_eth_tx_eo_wr *wr;
	2267	+ bool skip_eotx_wr = false;
	2268	+ struct tx_sw_desc *d;
	2269	+ struct sk_buff *skb;
	2270	+ int left, ret = 0;
	2271	+ u8 flits, ndesc;
	2272	+
	2273	+ eohw_txq = &adap->sge.eohw_txq[eosw_txq->hwqid];
	2274	+ spin_lock(&eohw_txq->lock);
	2275	+ reclaim_completed_tx_imm(&eohw_txq->q);
	2276	+
	2277	+ d = &eosw_txq->desc[eosw_txq->last_pidx];
	2278	+ skb = d->skb;
	2279	+ skb_tx_timestamp(skb);
	2280	+
	2281	+ wr = (struct fw_eth_tx_eo_wr *)&eohw_txq->q.desc[eohw_txq->q.pidx];
	2282	+ if (unlikely(eosw_txq->state != CXGB4_EO_STATE_ACTIVE &&
	2283	+ eosw_txq->last_pidx == eosw_txq->flowc_idx)) {
	2284	+ hdr_len = skb->len;
	2285	+ data_len = 0;
	2286	+ flits = DIV_ROUND_UP(hdr_len, 8);
	2287	+ if (eosw_txq->state == CXGB4_EO_STATE_FLOWC_OPEN_SEND)
	2288	+ next_state = CXGB4_EO_STATE_FLOWC_OPEN_REPLY;
	2289	+ else
	2290	+ next_state = CXGB4_EO_STATE_FLOWC_CLOSE_REPLY;
	2291	+ skip_eotx_wr = true;
	2292	+ } else {
	2293	+ hdr_len = eth_get_headlen(dev, skb->data, skb_headlen(skb));
	2294	+ data_len = skb->len - hdr_len;
	2295	+ flits = ethofld_calc_tx_flits(adap, skb, hdr_len);
	2296	+ }
	2297	+ ndesc = flits_to_desc(flits);
	2298	+ wrlen = flits * 8;
	2299	+ wrlen16 = DIV_ROUND_UP(wrlen, 16);
	2300	+
	2301	+ left = txq_avail(&eohw_txq->q) - ndesc;
	2302	+
	2303	+ /* If there are no descriptors left in hardware queues or no
	2304	+ * CPL credits left in software queues, then wait for them
	2305	+ * to come back and retry again. Note that we always request
	2306	+ * for credits update via interrupt for every half credits
	2307	+ * consumed. So, the interrupt will eventually restore the
	2308	+ * credits and invoke the Tx path again.
	2309	+ */
	2310	+ if (unlikely(left < 0 \|\| wrlen16 > eosw_txq->cred)) {
	2311	+ ret = -ENOMEM;
	2312	+ goto out_unlock;
	2313	+ }
	2314	+
	2315	+ if (unlikely(skip_eotx_wr)) {
	2316	+ start = (u64 *)wr;
	2317	+ eosw_txq->state = next_state;
	2318	+ eosw_txq->cred -= wrlen16;
	2319	+ eosw_txq->ncompl++;
	2320	+ eosw_txq->last_compl = 0;
	2321	+ goto write_wr_headers;
	2322	+ }
	2323	+
	2324	+ cpl = write_eo_wr(adap, eosw_txq, skb, wr, hdr_len, wrlen);
	2325	+ cntrl = hwcsum(adap->params.chip, skb);
	2326	+ if (skb_vlan_tag_present(skb))
	2327	+ cntrl \|= TXPKT_VLAN_VLD_F \| TXPKT_VLAN_V(skb_vlan_tag_get(skb));
	2328	+
	2329	+ cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT) \|
	2330	+ TXPKT_INTF_V(pi->tx_chan) \|
	2331	+ TXPKT_PF_V(adap->pf));
	2332	+ cpl->pack = 0;
	2333	+ cpl->len = cpu_to_be16(skb->len);
	2334	+ cpl->ctrl1 = cpu_to_be64(cntrl);
	2335	+
	2336	+ start = (u64 *)(cpl + 1);
	2337	+
	2338	+write_wr_headers:
	2339	+ sgl = (u64 )inline_tx_skb_header(skb, &eohw_txq->q, (void )start,
	2340	+ hdr_len);
	2341	+ if (data_len) {
	2342	+ ret = cxgb4_map_skb(adap->pdev_dev, skb, d->addr);
	2343	+ if (unlikely(ret)) {
	2344	+ memset(d->addr, 0, sizeof(d->addr));
	2345	+ eohw_txq->mapping_err++;
	2346	+ goto out_unlock;
	2347	+ }
	2348	+
	2349	+ end = (u64 *)wr + flits;
	2350	+ if (unlikely(start > sgl)) {
	2351	+ left = (u8 )end - (u8 )eohw_txq->q.stat;
	2352	+ end = (void *)eohw_txq->q.desc + left;
	2353	+ }
	2354	+
	2355	+ if (unlikely((u8 )sgl >= (u8 )eohw_txq->q.stat)) {
	2356	+ /* If current position is already at the end of the
	2357	+ * txq, reset the current to point to start of the queue
	2358	+ * and update the end ptr as well.
	2359	+ */
	2360	+ left = (u8 )end - (u8 )eohw_txq->q.stat;
	2361	+
	2362	+ end = (void *)eohw_txq->q.desc + left;
	2363	+ sgl = (void *)eohw_txq->q.desc;
	2364	+ }
	2365	+
	2366	+ cxgb4_write_sgl(skb, &eohw_txq->q, (void *)sgl, end, hdr_len,
	2367	+ d->addr);
	2368	+ }
	2369	+
	2370	+ if (skb_shinfo(skb)->gso_size) {
	2371	+ if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
	2372	+ eohw_txq->uso++;
	2373	+ else
	2374	+ eohw_txq->tso++;
	2375	+ eohw_txq->tx_cso += skb_shinfo(skb)->gso_segs;
	2376	+ } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
	2377	+ eohw_txq->tx_cso++;
	2378	+ }
	2379	+
	2380	+ if (skb_vlan_tag_present(skb))
	2381	+ eohw_txq->vlan_ins++;
	2382	+
	2383	+ txq_advance(&eohw_txq->q, ndesc);
	2384	+ cxgb4_ring_tx_db(adap, &eohw_txq->q, ndesc);
	2385	+ eosw_txq_advance_index(&eosw_txq->last_pidx, 1, eosw_txq->ndesc);
	2386	+
	2387	+out_unlock:
	2388	+ spin_unlock(&eohw_txq->lock);
	2389	+ return ret;
	2390	+}
	2391	+
	2392	+static void ethofld_xmit(struct net_device dev, struct sge_eosw_txq eosw_txq)
	2393	+{
	2394	+ struct sk_buff *skb;
	2395	+ int pktcount, ret;
	2396	+
	2397	+ switch (eosw_txq->state) {
	2398	+ case CXGB4_EO_STATE_ACTIVE:
	2399	+ case CXGB4_EO_STATE_FLOWC_OPEN_SEND:
	2400	+ case CXGB4_EO_STATE_FLOWC_CLOSE_SEND:
	2401	+ pktcount = eosw_txq->pidx - eosw_txq->last_pidx;
	2402	+ if (pktcount < 0)
	2403	+ pktcount += eosw_txq->ndesc;
	2404	+ break;
	2405	+ case CXGB4_EO_STATE_FLOWC_OPEN_REPLY:
	2406	+ case CXGB4_EO_STATE_FLOWC_CLOSE_REPLY:
	2407	+ case CXGB4_EO_STATE_CLOSED:
	2408	+ default:
	2409	+ return;
	2410	+ }
	2411	+
	2412	+ while (pktcount--) {
	2413	+ skb = eosw_txq_peek(eosw_txq);
	2414	+ if (!skb) {
	2415	+ eosw_txq_advance_index(&eosw_txq->last_pidx, 1,
	2416	+ eosw_txq->ndesc);
	2417	+ continue;
	2418	+ }
	2419	+
	2420	+ ret = ethofld_hard_xmit(dev, eosw_txq);
	2421	+ if (ret)
	2422	+ break;
	2423	+ }
	2424	+}
	2425	+
	2426	+static netdev_tx_t cxgb4_ethofld_xmit(struct sk_buff *skb,
	2427	+ struct net_device *dev)
	2428	+{
	2429	+ struct cxgb4_tc_port_mqprio *tc_port_mqprio;
	2430	+ struct port_info *pi = netdev2pinfo(dev);
	2431	+ struct adapter *adap = netdev2adap(dev);
	2432	+ struct sge_eosw_txq *eosw_txq;
	2433	+ u32 qid;
	2434	+ int ret;
	2435	+
	2436	+ ret = cxgb4_validate_skb(skb, dev, ETH_HLEN);
	2437	+ if (ret)
	2438	+ goto out_free;
	2439	+
	2440	+ tc_port_mqprio = &adap->tc_mqprio->port_mqprio[pi->port_id];
	2441	+ qid = skb_get_queue_mapping(skb) - pi->nqsets;
	2442	+ eosw_txq = &tc_port_mqprio->eosw_txq[qid];
	2443	+ spin_lock_bh(&eosw_txq->lock);
	2444	+ if (eosw_txq->state != CXGB4_EO_STATE_ACTIVE)
	2445	+ goto out_unlock;
	2446	+
	2447	+ ret = eosw_txq_enqueue(eosw_txq, skb);
	2448	+ if (ret)
	2449	+ goto out_unlock;
	2450	+
	2451	+ /* SKB is queued for processing until credits are available.
	2452	+ * So, call the destructor now and we'll free the skb later
	2453	+ * after it has been successfully transmitted.
	2454	+ */
	2455	+ skb_orphan(skb);
	2456	+
	2457	+ eosw_txq_advance(eosw_txq, 1);
	2458	+ ethofld_xmit(dev, eosw_txq);
	2459	+ spin_unlock_bh(&eosw_txq->lock);
	2460	+ return NETDEV_TX_OK;
	2461	+
	2462	+out_unlock:
	2463	+ spin_unlock_bh(&eosw_txq->lock);
	2464	+out_free:
	2465	+ dev_kfree_skb_any(skb);
	2466	+ return NETDEV_TX_OK;
	2467	+}
	2468	+
	2469	+netdev_tx_t t4_start_xmit(struct sk_buff skb, struct net_device dev)
	2470	+{
	2471	+ struct port_info *pi = netdev_priv(dev);
	2472	+ u16 qid = skb_get_queue_mapping(skb);
	2473	+
	2474	+ if (unlikely(pi->eth_flags & PRIV_FLAG_PORT_TX_VM))
	2475	+ return cxgb4_vf_eth_xmit(skb, dev);
	2476	+
	2477	+ if (unlikely(qid >= pi->nqsets))
	2478	+ return cxgb4_ethofld_xmit(skb, dev);
	2479	+
	2480	+ if (is_ptp_enabled(skb, dev)) {
	2481	+ struct adapter *adap = netdev2adap(dev);
	2482	+ netdev_tx_t ret;
	2483	+
	2484	+ spin_lock(&adap->ptp_lock);
	2485	+ ret = cxgb4_eth_xmit(skb, dev);
	2486	+ spin_unlock(&adap->ptp_lock);
	2487	+ return ret;
	2488	+ }
	2489	+
	2490	+ return cxgb4_eth_xmit(skb, dev);
	2491	+}
	2492	+
	2493	+static void eosw_txq_flush_pending_skbs(struct sge_eosw_txq *eosw_txq)
	2494	+{
	2495	+ int pktcount = eosw_txq->pidx - eosw_txq->last_pidx;
	2496	+ int pidx = eosw_txq->pidx;
	2497	+ struct sk_buff *skb;
	2498	+
	2499	+ if (!pktcount)
	2500	+ return;
	2501	+
	2502	+ if (pktcount < 0)
	2503	+ pktcount += eosw_txq->ndesc;
	2504	+
	2505	+ while (pktcount--) {
	2506	+ pidx--;
	2507	+ if (pidx < 0)
	2508	+ pidx += eosw_txq->ndesc;
	2509	+
	2510	+ skb = eosw_txq->desc[pidx].skb;
	2511	+ if (skb) {
	2512	+ dev_consume_skb_any(skb);
	2513	+ eosw_txq->desc[pidx].skb = NULL;
	2514	+ eosw_txq->inuse--;
	2515	+ }
	2516	+ }
	2517	+
	2518	+ eosw_txq->pidx = eosw_txq->last_pidx + 1;
	2519	+}
	2520	+
	2521	+/**
	2522	+ * cxgb4_ethofld_send_flowc - Send ETHOFLD flowc request to bind eotid to tc.
	2523	+ * @dev: netdevice
	2524	+ * @eotid: ETHOFLD tid to bind/unbind
	2525	+ * @tc: traffic class. If set to FW_SCHED_CLS_NONE, then unbinds the @eotid
	2526	+ *
	2527	+ * Send a FLOWC work request to bind an ETHOFLD TID to a traffic class.
	2528	+ * If @tc is set to FW_SCHED_CLS_NONE, then the @eotid is unbound from
	2529	+ * a traffic class.
	2530	+ */
	2531	+int cxgb4_ethofld_send_flowc(struct net_device *dev, u32 eotid, u32 tc)
	2532	+{
	2533	+ struct port_info *pi = netdev2pinfo(dev);
	2534	+ struct adapter *adap = netdev2adap(dev);
	2535	+ enum sge_eosw_state next_state;
	2536	+ struct sge_eosw_txq *eosw_txq;
	2537	+ u32 len, len16, nparams = 6;
	2538	+ struct fw_flowc_wr *flowc;
	2539	+ struct eotid_entry *entry;
	2540	+ struct sge_ofld_rxq *rxq;
	2541	+ struct sk_buff *skb;
	2542	+ int ret = 0;
	2543	+
	2544	+ len = struct_size(flowc, mnemval, nparams);
	2545	+ len16 = DIV_ROUND_UP(len, 16);
	2546	+
	2547	+ entry = cxgb4_lookup_eotid(&adap->tids, eotid);
	2548	+ if (!entry)
	2549	+ return -ENOMEM;
	2550	+
	2551	+ eosw_txq = (struct sge_eosw_txq *)entry->data;
	2552	+ if (!eosw_txq)
	2553	+ return -ENOMEM;
	2554	+
	2555	+ if (!(adap->flags & CXGB4_FW_OK)) {
	2556	+ /* Don't stall caller when access to FW is lost */
	2557	+ complete(&eosw_txq->completion);
	2558	+ return -EIO;
	2559	+ }
	2560	+
	2561	+ skb = alloc_skb(len, GFP_KERNEL);
	2562	+ if (!skb)
	2563	+ return -ENOMEM;
	2564	+
	2565	+ spin_lock_bh(&eosw_txq->lock);
	2566	+ if (tc != FW_SCHED_CLS_NONE) {
	2567	+ if (eosw_txq->state != CXGB4_EO_STATE_CLOSED)
	2568	+ goto out_free_skb;
	2569	+
	2570	+ next_state = CXGB4_EO_STATE_FLOWC_OPEN_SEND;
	2571	+ } else {
	2572	+ if (eosw_txq->state != CXGB4_EO_STATE_ACTIVE)
	2573	+ goto out_free_skb;
	2574	+
	2575	+ next_state = CXGB4_EO_STATE_FLOWC_CLOSE_SEND;
	2576	+ }
	2577	+
	2578	+ flowc = __skb_put(skb, len);
	2579	+ memset(flowc, 0, len);
	2580	+
	2581	+ rxq = &adap->sge.eohw_rxq[eosw_txq->hwqid];
	2582	+ flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(len16) \|
	2583	+ FW_WR_FLOWID_V(eosw_txq->hwtid));
	2584	+ flowc->op_to_nparams = cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) \|
	2585	+ FW_FLOWC_WR_NPARAMS_V(nparams) \|
	2586	+ FW_WR_COMPL_V(1));
	2587	+ flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
	2588	+ flowc->mnemval[0].val = cpu_to_be32(FW_PFVF_CMD_PFN_V(adap->pf));
	2589	+ flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
	2590	+ flowc->mnemval[1].val = cpu_to_be32(pi->tx_chan);
	2591	+ flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
	2592	+ flowc->mnemval[2].val = cpu_to_be32(pi->tx_chan);
	2593	+ flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
	2594	+ flowc->mnemval[3].val = cpu_to_be32(rxq->rspq.abs_id);
	2595	+ flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS;
	2596	+ flowc->mnemval[4].val = cpu_to_be32(tc);
	2597	+ flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_EOSTATE;
	2598	+ flowc->mnemval[5].val = cpu_to_be32(tc == FW_SCHED_CLS_NONE ?
	2599	+ FW_FLOWC_MNEM_EOSTATE_CLOSING :
	2600	+ FW_FLOWC_MNEM_EOSTATE_ESTABLISHED);
	2601	+
	2602	+ /* Free up any pending skbs to ensure there's room for
	2603	+ * termination FLOWC.
	2604	+ */
	2605	+ if (tc == FW_SCHED_CLS_NONE)
	2606	+ eosw_txq_flush_pending_skbs(eosw_txq);
	2607	+
	2608	+ ret = eosw_txq_enqueue(eosw_txq, skb);
	2609	+ if (ret)
	2610	+ goto out_free_skb;
	2611	+
	2612	+ eosw_txq->state = next_state;
	2613	+ eosw_txq->flowc_idx = eosw_txq->pidx;
	2614	+ eosw_txq_advance(eosw_txq, 1);
	2615	+ ethofld_xmit(dev, eosw_txq);
	2616	+
	2617	+ spin_unlock_bh(&eosw_txq->lock);
	2618	+ return 0;
	2619	+
	2620	+out_free_skb:
	2621	+ dev_consume_skb_any(skb);
	2622	+ spin_unlock_bh(&eosw_txq->lock);
	2623	+ return ret;
1936	2624	}
1937	2625
1938	2626	/**
..	..	@@ -1964,6 +2652,84 @@
1964	2652	q->q.stops++;
1965	2653	q->full = 1;
1966	2654	}
	2655	+}
	2656	+
	2657	+#define CXGB4_SELFTEST_LB_STR "CHELSIO_SELFTEST"
	2658	+
	2659	+int cxgb4_selftest_lb_pkt(struct net_device *netdev)
	2660	+{
	2661	+ struct port_info *pi = netdev_priv(netdev);
	2662	+ struct adapter *adap = pi->adapter;
	2663	+ struct cxgb4_ethtool_lb_test *lb;
	2664	+ int ret, i = 0, pkt_len, credits;
	2665	+ struct fw_eth_tx_pkt_wr *wr;
	2666	+ struct cpl_tx_pkt_core *cpl;
	2667	+ u32 ctrl0, ndesc, flits;
	2668	+ struct sge_eth_txq *q;
	2669	+ u8 *sgl;
	2670	+
	2671	+ pkt_len = ETH_HLEN + sizeof(CXGB4_SELFTEST_LB_STR);
	2672	+
	2673	+ flits = DIV_ROUND_UP(pkt_len + sizeof(cpl) + sizeof(wr),
	2674	+ sizeof(__be64));
	2675	+ ndesc = flits_to_desc(flits);
	2676	+
	2677	+ lb = &pi->ethtool_lb;
	2678	+ lb->loopback = 1;
	2679	+
	2680	+ q = &adap->sge.ethtxq[pi->first_qset];
	2681	+ __netif_tx_lock(q->txq, smp_processor_id());
	2682	+
	2683	+ reclaim_completed_tx(adap, &q->q, -1, true);
	2684	+ credits = txq_avail(&q->q) - ndesc;
	2685	+ if (unlikely(credits < 0)) {
	2686	+ __netif_tx_unlock(q->txq);
	2687	+ return -ENOMEM;
	2688	+ }
	2689	+
	2690	+ wr = (void *)&q->q.desc[q->q.pidx];
	2691	+ memset(wr, 0, sizeof(struct tx_desc));
	2692	+
	2693	+ wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) \|
	2694	+ FW_WR_IMMDLEN_V(pkt_len +
	2695	+ sizeof(*cpl)));
	2696	+ wr->equiq_to_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2)));
	2697	+ wr->r3 = cpu_to_be64(0);
	2698	+
	2699	+ cpl = (void *)(wr + 1);
	2700	+ sgl = (u8 *)(cpl + 1);
	2701	+
	2702	+ ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) \| TXPKT_PF_V(adap->pf) \|
	2703	+ TXPKT_INTF_V(pi->tx_chan + 4);
	2704	+
	2705	+ cpl->ctrl0 = htonl(ctrl0);
	2706	+ cpl->pack = htons(0);
	2707	+ cpl->len = htons(pkt_len);
	2708	+ cpl->ctrl1 = cpu_to_be64(TXPKT_L4CSUM_DIS_F \| TXPKT_IPCSUM_DIS_F);
	2709	+
	2710	+ eth_broadcast_addr(sgl);
	2711	+ i += ETH_ALEN;
	2712	+ ether_addr_copy(&sgl[i], netdev->dev_addr);
	2713	+ i += ETH_ALEN;
	2714	+
	2715	+ snprintf(&sgl[i], sizeof(CXGB4_SELFTEST_LB_STR), "%s",
	2716	+ CXGB4_SELFTEST_LB_STR);
	2717	+
	2718	+ init_completion(&lb->completion);
	2719	+ txq_advance(&q->q, ndesc);
	2720	+ cxgb4_ring_tx_db(adap, &q->q, ndesc);
	2721	+ __netif_tx_unlock(q->txq);
	2722	+
	2723	+ /* wait for the pkt to return */
	2724	+ ret = wait_for_completion_timeout(&lb->completion, 10 * HZ);
	2725	+ if (!ret)
	2726	+ ret = -ETIMEDOUT;
	2727	+ else
	2728	+ ret = lb->result;
	2729	+
	2730	+ lb->loopback = 0;
	2731	+
	2732	+ return ret;
1967	2733	}
1968	2734
1969	2735	/**
..	..	@@ -2011,15 +2777,15 @@
2011	2777
2012	2778	/**
2013	2779	* restart_ctrlq - restart a suspended control queue
2014		- * @data: the control queue to restart
	2780	+ * @t: pointer to the tasklet associated with this handler
2015	2781	*
2016	2782	* Resumes transmission on a suspended Tx control queue.
2017	2783	*/
2018		-static void restart_ctrlq(unsigned long data)
	2784	+static void restart_ctrlq(struct tasklet_struct *t)
2019	2785	{
2020	2786	struct sk_buff *skb;
2021	2787	unsigned int written = 0;
2022		- struct sge_ctrl_txq q = (struct sge_ctrl_txq )data;
	2788	+ struct sge_ctrl_txq *q = from_tasklet(q, t, qresume_tsk);
2023	2789
2024	2790	spin_lock(&q->sendq.lock);
2025	2791	reclaim_completed_tx_imm(&q->q);
..	..	@@ -2126,7 +2892,6 @@
2126	2892
2127	2893	/**
2128	2894	* txq_stop_maperr - stop a Tx queue due to I/O MMU exhaustion
2129		- * @adap: the adapter
2130	2895	* @q: the queue to stop
2131	2896	*
2132	2897	* Mark a Tx queue stopped due to I/O MMU exhaustion and resulting
..	..	@@ -2175,6 +2940,7 @@
2175	2940	* is ever running at a time ...
2176	2941	*/
2177	2942	static void service_ofldq(struct sge_uld_txq *q)
	2943	+ __must_hold(&q->sendq.lock)
2178	2944	{
2179	2945	u64 pos, before, *end;
2180	2946	int credits;
..	..	@@ -2317,13 +3083,13 @@
2317	3083
2318	3084	/**
2319	3085	* restart_ofldq - restart a suspended offload queue
2320		- * @data: the offload queue to restart
	3086	+ * @t: pointer to the tasklet associated with this handler
2321	3087	*
2322	3088	* Resumes transmission on a suspended Tx offload queue.
2323	3089	*/
2324		-static void restart_ofldq(unsigned long data)
	3090	+static void restart_ofldq(struct tasklet_struct *t)
2325	3091	{
2326		- struct sge_uld_txq q = (struct sge_uld_txq )data;
	3092	+ struct sge_uld_txq *q = from_tasklet(q, t, qresume_tsk);
2327	3093
2328	3094	spin_lock(&q->sendq.lock);
2329	3095	q->full = 0; /* the queue actually is completely empty now */
..	..	@@ -2721,7 +3487,7 @@
2721	3487
2722	3488	/**
2723	3489	* t4_systim_to_hwstamp - read hardware time stamp
2724		- * @adap: the adapter
	3490	+ * @adapter: the adapter
2725	3491	* @skb: the packet
2726	3492	*
2727	3493	* Read Time Stamp from MPS packet and insert in skb which
..	..	@@ -2755,8 +3521,9 @@
2755	3521
2756	3522	/**
2757	3523	* t4_rx_hststamp - Recv PTP Event Message
2758		- * @adap: the adapter
	3524	+ * @adapter: the adapter
2759	3525	* @rsp: the response queue descriptor holding the RX_PKT message
	3526	+ * @rxq: the response queue holding the RX_PKT message
2760	3527	* @skb: the packet
2761	3528	*
2762	3529	* PTP enabled and MPS packet, read HW timestamp
..	..	@@ -2780,7 +3547,7 @@
2780	3547
2781	3548	/**
2782	3549	* t4_tx_hststamp - Loopback PTP Transmit Event Message
2783		- * @adap: the adapter
	3550	+ * @adapter: the adapter
2784	3551	* @skb: the packet
2785	3552	* @dev: the ingress net device
2786	3553	*
..	..	@@ -2797,6 +3564,79 @@
2797	3564	return 0;
2798	3565	}
2799	3566	return 1;
	3567	+}
	3568	+
	3569	+/**
	3570	+ * t4_tx_completion_handler - handle CPL_SGE_EGR_UPDATE messages
	3571	+ * @rspq: Ethernet RX Response Queue associated with Ethernet TX Queue
	3572	+ * @rsp: Response Entry pointer into Response Queue
	3573	+ * @gl: Gather List pointer
	3574	+ *
	3575	+ * For adapters which support the SGE Doorbell Queue Timer facility,
	3576	+ * we configure the Ethernet TX Queues to send CIDX Updates to the
	3577	+ * Associated Ethernet RX Response Queue with CPL_SGE_EGR_UPDATE
	3578	+ * messages. This adds a small load to PCIe Link RX bandwidth and,
	3579	+ * potentially, higher CPU Interrupt load, but allows us to respond
	3580	+ * much more quickly to the CIDX Updates. This is important for
	3581	+ * Upper Layer Software which isn't willing to have a large amount
	3582	+ * of TX Data outstanding before receiving DMA Completions.
	3583	+ */
	3584	+static void t4_tx_completion_handler(struct sge_rspq *rspq,
	3585	+ const __be64 *rsp,
	3586	+ const struct pkt_gl *gl)
	3587	+{
	3588	+ u8 opcode = ((const struct rss_header *)rsp)->opcode;
	3589	+ struct port_info *pi = netdev_priv(rspq->netdev);
	3590	+ struct adapter *adapter = rspq->adap;
	3591	+ struct sge *s = &adapter->sge;
	3592	+ struct sge_eth_txq *txq;
	3593	+
	3594	+ /* skip RSS header */
	3595	+ rsp++;
	3596	+
	3597	+ /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
	3598	+ */
	3599	+ if (unlikely(opcode == CPL_FW4_MSG &&
	3600	+ ((const struct cpl_fw4_msg *)rsp)->type ==
	3601	+ FW_TYPE_RSSCPL)) {
	3602	+ rsp++;
	3603	+ opcode = ((const struct rss_header *)rsp)->opcode;
	3604	+ rsp++;
	3605	+ }
	3606	+
	3607	+ if (unlikely(opcode != CPL_SGE_EGR_UPDATE)) {
	3608	+ pr_info("%s: unexpected FW4/CPL %#x on Rx queue\n",
	3609	+ __func__, opcode);
	3610	+ return;
	3611	+ }
	3612	+
	3613	+ txq = &s->ethtxq[pi->first_qset + rspq->idx];
	3614	+ t4_sge_eth_txq_egress_update(adapter, txq, -1);
	3615	+}
	3616	+
	3617	+static int cxgb4_validate_lb_pkt(struct port_info pi, const struct pkt_gl si)
	3618	+{
	3619	+ struct adapter *adap = pi->adapter;
	3620	+ struct cxgb4_ethtool_lb_test *lb;
	3621	+ struct sge *s = &adap->sge;
	3622	+ struct net_device *netdev;
	3623	+ u8 *data;
	3624	+ int i;
	3625	+
	3626	+ netdev = adap->port[pi->port_id];
	3627	+ lb = &pi->ethtool_lb;
	3628	+ data = si->va + s->pktshift;
	3629	+
	3630	+ i = ETH_ALEN;
	3631	+ if (!ether_addr_equal(data + i, netdev->dev_addr))
	3632	+ return -1;
	3633	+
	3634	+ i += ETH_ALEN;
	3635	+ if (strcmp(&data[i], CXGB4_SELFTEST_LB_STR))
	3636	+ lb->result = -EIO;
	3637	+
	3638	+ complete(&lb->completion);
	3639	+ return 0;
2800	3640	}
2801	3641
2802	3642	/**
..	..	@@ -2822,6 +3662,16 @@
2822	3662	struct port_info *pi;
2823	3663	int ret = 0;
2824	3664
	3665	+ pi = netdev_priv(q->netdev);
	3666	+ /* If we're looking at TX Queue CIDX Update, handle that separately
	3667	+ * and return.
	3668	+ */
	3669	+ if (unlikely(((u8 )rsp == CPL_FW4_MSG) \|\|
	3670	+ ((u8 )rsp == CPL_SGE_EGR_UPDATE))) {
	3671	+ t4_tx_completion_handler(q, rsp, si);
	3672	+ return 0;
	3673	+ }
	3674	+
2825	3675	if (unlikely((u8 )rsp == cpl_trace_pkt))
2826	3676	return handle_trace_pkt(q->adap, si);
2827	3677
..	..	@@ -2836,6 +3686,16 @@
2836	3686
2837	3687	csum_ok = pkt->csum_calc && !err_vec &&
2838	3688	(q->netdev->features & NETIF_F_RXCSUM);
	3689	+
	3690	+ if (err_vec)
	3691	+ rxq->stats.bad_rx_pkts++;
	3692	+
	3693	+ if (unlikely(pi->ethtool_lb.loopback && pkt->iff >= NCHAN)) {
	3694	+ ret = cxgb4_validate_lb_pkt(pi, si);
	3695	+ if (!ret)
	3696	+ return 0;
	3697	+ }
	3698	+
2839	3699	if (((pkt->l2info & htonl(RXF_TCP_F)) \|\|
2840	3700	tnl_hdr_len) &&
2841	3701	(q->netdev->features & NETIF_F_GRO) && csum_ok && !pkt->ip_frag) {
..	..	@@ -2849,7 +3709,6 @@
2849	3709	rxq->stats.rx_drops++;
2850	3710	return 0;
2851	3711	}
2852		- pi = netdev_priv(q->netdev);
2853	3712
2854	3713	/* Handle PTP Event Rx packet */
2855	3714	if (unlikely(pi->ptp_enable)) {
..	..	@@ -3150,6 +4009,113 @@
3150	4009	return work_done;
3151	4010	}
3152	4011
	4012	+void cxgb4_ethofld_restart(struct tasklet_struct *t)
	4013	+{
	4014	+ struct sge_eosw_txq *eosw_txq = from_tasklet(eosw_txq, t,
	4015	+ qresume_tsk);
	4016	+ int pktcount;
	4017	+
	4018	+ spin_lock(&eosw_txq->lock);
	4019	+ pktcount = eosw_txq->cidx - eosw_txq->last_cidx;
	4020	+ if (pktcount < 0)
	4021	+ pktcount += eosw_txq->ndesc;
	4022	+
	4023	+ if (pktcount) {
	4024	+ cxgb4_eosw_txq_free_desc(netdev2adap(eosw_txq->netdev),
	4025	+ eosw_txq, pktcount);
	4026	+ eosw_txq->inuse -= pktcount;
	4027	+ }
	4028	+
	4029	+ /* There may be some packets waiting for completions. So,
	4030	+ * attempt to send these packets now.
	4031	+ */
	4032	+ ethofld_xmit(eosw_txq->netdev, eosw_txq);
	4033	+ spin_unlock(&eosw_txq->lock);
	4034	+}
	4035	+
	4036	+/* cxgb4_ethofld_rx_handler - Process ETHOFLD Tx completions
	4037	+ * @q: the response queue that received the packet
	4038	+ * @rsp: the response queue descriptor holding the CPL message
	4039	+ * @si: the gather list of packet fragments
	4040	+ *
	4041	+ * Process a ETHOFLD Tx completion. Increment the cidx here, but
	4042	+ * free up the descriptors in a tasklet later.
	4043	+ */
	4044	+int cxgb4_ethofld_rx_handler(struct sge_rspq q, const __be64 rsp,
	4045	+ const struct pkt_gl *si)
	4046	+{
	4047	+ u8 opcode = ((const struct rss_header *)rsp)->opcode;
	4048	+
	4049	+ /* skip RSS header */
	4050	+ rsp++;
	4051	+
	4052	+ if (opcode == CPL_FW4_ACK) {
	4053	+ const struct cpl_fw4_ack *cpl;
	4054	+ struct sge_eosw_txq *eosw_txq;
	4055	+ struct eotid_entry *entry;
	4056	+ struct sk_buff *skb;
	4057	+ u32 hdr_len, eotid;
	4058	+ u8 flits, wrlen16;
	4059	+ int credits;
	4060	+
	4061	+ cpl = (const struct cpl_fw4_ack *)rsp;
	4062	+ eotid = CPL_FW4_ACK_FLOWID_G(ntohl(OPCODE_TID(cpl))) -
	4063	+ q->adap->tids.eotid_base;
	4064	+ entry = cxgb4_lookup_eotid(&q->adap->tids, eotid);
	4065	+ if (!entry)
	4066	+ goto out_done;
	4067	+
	4068	+ eosw_txq = (struct sge_eosw_txq *)entry->data;
	4069	+ if (!eosw_txq)
	4070	+ goto out_done;
	4071	+
	4072	+ spin_lock(&eosw_txq->lock);
	4073	+ credits = cpl->credits;
	4074	+ while (credits > 0) {
	4075	+ skb = eosw_txq->desc[eosw_txq->cidx].skb;
	4076	+ if (!skb)
	4077	+ break;
	4078	+
	4079	+ if (unlikely((eosw_txq->state ==
	4080	+ CXGB4_EO_STATE_FLOWC_OPEN_REPLY \|\|
	4081	+ eosw_txq->state ==
	4082	+ CXGB4_EO_STATE_FLOWC_CLOSE_REPLY) &&
	4083	+ eosw_txq->cidx == eosw_txq->flowc_idx)) {
	4084	+ flits = DIV_ROUND_UP(skb->len, 8);
	4085	+ if (eosw_txq->state ==
	4086	+ CXGB4_EO_STATE_FLOWC_OPEN_REPLY)
	4087	+ eosw_txq->state = CXGB4_EO_STATE_ACTIVE;
	4088	+ else
	4089	+ eosw_txq->state = CXGB4_EO_STATE_CLOSED;
	4090	+ complete(&eosw_txq->completion);
	4091	+ } else {
	4092	+ hdr_len = eth_get_headlen(eosw_txq->netdev,
	4093	+ skb->data,
	4094	+ skb_headlen(skb));
	4095	+ flits = ethofld_calc_tx_flits(q->adap, skb,
	4096	+ hdr_len);
	4097	+ }
	4098	+ eosw_txq_advance_index(&eosw_txq->cidx, 1,
	4099	+ eosw_txq->ndesc);
	4100	+ wrlen16 = DIV_ROUND_UP(flits * 8, 16);
	4101	+ credits -= wrlen16;
	4102	+ }
	4103	+
	4104	+ eosw_txq->cred += cpl->credits;
	4105	+ eosw_txq->ncompl--;
	4106	+
	4107	+ spin_unlock(&eosw_txq->lock);
	4108	+
	4109	+ /* Schedule a tasklet to reclaim SKBs and restart ETHOFLD Tx,
	4110	+ * if there were packets waiting for completion.
	4111	+ */
	4112	+ tasklet_schedule(&eosw_txq->qresume_tsk);
	4113	+ }
	4114	+
	4115	+out_done:
	4116	+ return 0;
	4117	+}
	4118	+
3153	4119	/*
3154	4120	* The MSI-X interrupt handler for an SGE response queue.
3155	4121	*/
..	..	@@ -3214,7 +4180,7 @@
3214	4180	{
3215	4181	struct adapter *adap = cookie;
3216	4182
3217		- if (adap->flags & MASTER_PF)
	4183	+ if (adap->flags & CXGB4_MASTER_PF)
3218	4184	t4_slow_intr_handler(adap);
3219	4185	process_intrq(adap);
3220	4186	return IRQ_HANDLED;
..	..	@@ -3230,7 +4196,7 @@
3230	4196	struct adapter *adap = cookie;
3231	4197
3232	4198	t4_write_reg(adap, MYPF_REG(PCIE_PF_CLI_A), 0);
3233		- if (((adap->flags & MASTER_PF) && t4_slow_intr_handler(adap)) \|
	4199	+ if (((adap->flags & CXGB4_MASTER_PF) && t4_slow_intr_handler(adap)) \|
3234	4200	process_intrq(adap))
3235	4201	return IRQ_HANDLED;
3236	4202	return IRQ_NONE; /* probably shared interrupt */
..	..	@@ -3245,9 +4211,9 @@
3245	4211	*/
3246	4212	irq_handler_t t4_intr_handler(struct adapter *adap)
3247	4213	{
3248		- if (adap->flags & USING_MSIX)
	4214	+ if (adap->flags & CXGB4_USING_MSIX)
3249	4215	return t4_sge_intr_msix;
3250		- if (adap->flags & USING_MSI)
	4216	+ if (adap->flags & CXGB4_USING_MSI)
3251	4217	return t4_intr_msi;
3252	4218	return t4_intr_intx;
3253	4219	}
..	..	@@ -3280,7 +4246,7 @@
3280	4246	* global Master PF activities like checking for chip ingress stalls,
3281	4247	* etc.
3282	4248	*/
3283		- if (!(adap->flags & MASTER_PF))
	4249	+ if (!(adap->flags & CXGB4_MASTER_PF))
3284	4250	goto done;
3285	4251
3286	4252	t4_idma_monitor(adap, &s->idma_monitor, HZ, RX_QCHECK_PERIOD);
..	..	@@ -3291,10 +4257,10 @@
3291	4257
3292	4258	static void sge_tx_timer_cb(struct timer_list *t)
3293	4259	{
3294		- unsigned long m;
3295		- unsigned int i, budget;
3296	4260	struct adapter *adap = from_timer(adap, t, sge.tx_timer);
3297	4261	struct sge *s = &adap->sge;
	4262	+ unsigned long m, period;
	4263	+ unsigned int i, budget;
3298	4264
3299	4265	for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++)
3300	4266	for (m = s->txq_maperr[i]; m; m &= m - 1) {
..	..	@@ -3322,29 +4288,29 @@
3322	4288	budget = MAX_TIMER_TX_RECLAIM;
3323	4289	i = s->ethtxq_rover;
3324	4290	do {
3325		- struct sge_eth_txq *q = &s->ethtxq[i];
3326		-
3327		- if (q->q.in_use &&
3328		- time_after_eq(jiffies, q->txq->trans_start + HZ / 100) &&
3329		- __netif_tx_trylock(q->txq)) {
3330		- int avail = reclaimable(&q->q);
3331		-
3332		- if (avail) {
3333		- if (avail > budget)
3334		- avail = budget;
3335		-
3336		- free_tx_desc(adap, &q->q, avail, true);
3337		- q->q.in_use -= avail;
3338		- budget -= avail;
3339		- }
3340		- __netif_tx_unlock(q->txq);
3341		- }
	4291	+ budget -= t4_sge_eth_txq_egress_update(adap, &s->ethtxq[i],
	4292	+ budget);
	4293	+ if (!budget)
	4294	+ break;
3342	4295
3343	4296	if (++i >= s->ethqsets)
3344	4297	i = 0;
3345		- } while (budget && i != s->ethtxq_rover);
	4298	+ } while (i != s->ethtxq_rover);
3346	4299	s->ethtxq_rover = i;
3347		- mod_timer(&s->tx_timer, jiffies + (budget ? TX_QCHECK_PERIOD : 2));
	4300	+
	4301	+ if (budget == 0) {
	4302	+ /* If we found too many reclaimable packets schedule a timer
	4303	+ * in the near future to continue where we left off.
	4304	+ */
	4305	+ period = 2;
	4306	+ } else {
	4307	+ /* We reclaimed all reclaimable TX Descriptors, so reschedule
	4308	+ * at the normal period.
	4309	+ */
	4310	+ period = TX_QCHECK_PERIOD;
	4311	+ }
	4312	+
	4313	+ mod_timer(&s->tx_timer, jiffies + period);
3348	4314	}
3349	4315
3350	4316	/**
..	..	@@ -3388,7 +4354,7 @@
3388	4354	struct fw_iq_cmd c;
3389	4355	struct sge *s = &adap->sge;
3390	4356	struct port_info *pi = netdev_priv(dev);
3391		- int relaxed = !(adap->flags & ROOT_NO_RELAXED_ORDERING);
	4357	+ int relaxed = !(adap->flags & CXGB4_ROOT_NO_RELAXED_ORDERING);
3392	4358
3393	4359	/* Size needs to be multiple of 16, including status entry. */
3394	4360	iq->size = roundup(iq->size, 16);
..	..	@@ -3423,7 +4389,8 @@
3423	4389	: FW_IQ_IQTYPE_OFLD));
3424	4390
3425	4391	if (fl) {
3426		- enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip);
	4392	+ unsigned int chip_ver =
	4393	+ CHELSIO_CHIP_VERSION(adap->params.chip);
3427	4394
3428	4395	/* Allocate the ring for the hardware free list (with space
3429	4396	* for its status page) along with the associated software
..	..	@@ -3461,10 +4428,10 @@
3461	4428	* the smaller 64-byte value there).
3462	4429	*/
3463	4430	c.fl0dcaen_to_fl0cidxfthresh =
3464		- htons(FW_IQ_CMD_FL0FBMIN_V(chip <= CHELSIO_T5 ?
	4431	+ htons(FW_IQ_CMD_FL0FBMIN_V(chip_ver <= CHELSIO_T5 ?
3465	4432	FETCHBURSTMIN_128B_X :
3466		- FETCHBURSTMIN_64B_X) \|
3467		- FW_IQ_CMD_FL0FBMAX_V((chip <= CHELSIO_T5) ?
	4433	+ FETCHBURSTMIN_64B_T6_X) \|
	4434	+ FW_IQ_CMD_FL0FBMAX_V((chip_ver <= CHELSIO_T5) ?
3468	4435	FETCHBURSTMAX_512B_X :
3469	4436	FETCHBURSTMAX_256B_X));
3470	4437	c.fl0size = htons(flsz);
..	..	@@ -3586,14 +4553,24 @@
3586	4553	adap->sge.egr_map[id - adap->sge.egr_start] = q;
3587	4554	}
3588	4555
	4556	+/**
	4557	+ * t4_sge_alloc_eth_txq - allocate an Ethernet TX Queue
	4558	+ * @adap: the adapter
	4559	+ * @txq: the SGE Ethernet TX Queue to initialize
	4560	+ * @dev: the Linux Network Device
	4561	+ * @netdevq: the corresponding Linux TX Queue
	4562	+ * @iqid: the Ingress Queue to which to deliver CIDX Update messages
	4563	+ * @dbqt: whether this TX Queue will use the SGE Doorbell Queue Timers
	4564	+ */
3589	4565	int t4_sge_alloc_eth_txq(struct adapter adap, struct sge_eth_txq txq,
3590	4566	struct net_device dev, struct netdev_queue netdevq,
3591		- unsigned int iqid)
	4567	+ unsigned int iqid, u8 dbqt)
3592	4568	{
3593		- int ret, nentries;
3594		- struct fw_eq_eth_cmd c;
3595		- struct sge *s = &adap->sge;
	4569	+ unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
3596	4570	struct port_info *pi = netdev_priv(dev);
	4571	+ struct sge *s = &adap->sge;
	4572	+ struct fw_eq_eth_cmd c;
	4573	+ int ret, nentries;
3597	4574
3598	4575	/* Add status entries */
3599	4576	nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
..	..	@@ -3612,18 +4589,42 @@
3612	4589	FW_EQ_ETH_CMD_VFN_V(0));
3613	4590	c.alloc_to_len16 = htonl(FW_EQ_ETH_CMD_ALLOC_F \|
3614	4591	FW_EQ_ETH_CMD_EQSTART_F \| FW_LEN16(c));
3615		- c.viid_pkd = htonl(FW_EQ_ETH_CMD_AUTOEQUEQE_F \|
3616		- FW_EQ_ETH_CMD_VIID_V(pi->viid));
	4592	+
	4593	+ /* For TX Ethernet Queues using the SGE Doorbell Queue Timer
	4594	+ * mechanism, we use Ingress Queue messages for Hardware Consumer
	4595	+ * Index Updates on the TX Queue. Otherwise we have the Hardware
	4596	+ * write the CIDX Updates into the Status Page at the end of the
	4597	+ * TX Queue.
	4598	+ */
	4599	+ c.autoequiqe_to_viid = htonl(FW_EQ_ETH_CMD_AUTOEQUEQE_F \|
	4600	+ FW_EQ_ETH_CMD_VIID_V(pi->viid));
	4601	+
3617	4602	c.fetchszm_to_iqid =
3618	4603	htonl(FW_EQ_ETH_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) \|
3619	4604	FW_EQ_ETH_CMD_PCIECHN_V(pi->tx_chan) \|
3620	4605	FW_EQ_ETH_CMD_FETCHRO_F \| FW_EQ_ETH_CMD_IQID_V(iqid));
	4606	+
	4607	+ /* Note that the CIDX Flush Threshold should match MAX_TX_RECLAIM. */
3621	4608	c.dcaen_to_eqsize =
3622		- htonl(FW_EQ_ETH_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) \|
	4609	+ htonl(FW_EQ_ETH_CMD_FBMIN_V(chip_ver <= CHELSIO_T5
	4610	+ ? FETCHBURSTMIN_64B_X
	4611	+ : FETCHBURSTMIN_64B_T6_X) \|
3623	4612	FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) \|
3624	4613	FW_EQ_ETH_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) \|
3625	4614	FW_EQ_ETH_CMD_EQSIZE_V(nentries));
	4615	+
3626	4616	c.eqaddr = cpu_to_be64(txq->q.phys_addr);
	4617	+
	4618	+ /* If we're using the SGE Doorbell Queue Timer mechanism, pass in the
	4619	+ * currently configured Timer Index. THis can be changed later via an
	4620	+ * ethtool -C tx-usecs {Timer Val} command. Note that the SGE
	4621	+ * Doorbell Queue mode is currently automatically enabled in the
	4622	+ * Firmware by setting either AUTOEQUEQE or AUTOEQUIQE ...
	4623	+ */
	4624	+ if (dbqt)
	4625	+ c.timeren_timerix =
	4626	+ cpu_to_be32(FW_EQ_ETH_CMD_TIMEREN_F \|
	4627	+ FW_EQ_ETH_CMD_TIMERIX_V(txq->dbqtimerix));
3627	4628
3628	4629	ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3629	4630	if (ret) {
..	..	@@ -3639,8 +4640,13 @@
3639	4640	txq->q.q_type = CXGB4_TXQ_ETH;
3640	4641	init_txq(adap, &txq->q, FW_EQ_ETH_CMD_EQID_G(ntohl(c.eqid_pkd)));
3641	4642	txq->txq = netdevq;
3642		- txq->tso = txq->tx_cso = txq->vlan_ins = 0;
	4643	+ txq->tso = 0;
	4644	+ txq->uso = 0;
	4645	+ txq->tx_cso = 0;
	4646	+ txq->vlan_ins = 0;
3643	4647	txq->mapping_err = 0;
	4648	+ txq->dbqt = dbqt;
	4649	+
3644	4650	return 0;
3645	4651	}
3646	4652
..	..	@@ -3648,10 +4654,11 @@
3648	4654	struct net_device *dev, unsigned int iqid,
3649	4655	unsigned int cmplqid)
3650	4656	{
3651		- int ret, nentries;
3652		- struct fw_eq_ctrl_cmd c;
3653		- struct sge *s = &adap->sge;
	4657	+ unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
3654	4658	struct port_info *pi = netdev_priv(dev);
	4659	+ struct sge *s = &adap->sge;
	4660	+ struct fw_eq_ctrl_cmd c;
	4661	+ int ret, nentries;
3655	4662
3656	4663	/* Add status entries */
3657	4664	nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
..	..	@@ -3675,7 +4682,9 @@
3675	4682	FW_EQ_CTRL_CMD_PCIECHN_V(pi->tx_chan) \|
3676	4683	FW_EQ_CTRL_CMD_FETCHRO_F \| FW_EQ_CTRL_CMD_IQID_V(iqid));
3677	4684	c.dcaen_to_eqsize =
3678		- htonl(FW_EQ_CTRL_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) \|
	4685	+ htonl(FW_EQ_CTRL_CMD_FBMIN_V(chip_ver <= CHELSIO_T5
	4686	+ ? FETCHBURSTMIN_64B_X
	4687	+ : FETCHBURSTMIN_64B_T6_X) \|
3679	4688	FW_EQ_CTRL_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) \|
3680	4689	FW_EQ_CTRL_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) \|
3681	4690	FW_EQ_CTRL_CMD_EQSIZE_V(nentries));
..	..	@@ -3694,7 +4703,7 @@
3694	4703	init_txq(adap, &txq->q, FW_EQ_CTRL_CMD_EQID_G(ntohl(c.cmpliqid_eqid)));
3695	4704	txq->adap = adap;
3696	4705	skb_queue_head_init(&txq->sendq);
3697		- tasklet_init(&txq->qresume_tsk, restart_ctrlq, (unsigned long)txq);
	4706	+ tasklet_setup(&txq->qresume_tsk, restart_ctrlq);
3698	4707	txq->full = 0;
3699	4708	return 0;
3700	4709	}
..	..	@@ -3711,29 +4720,30 @@
3711	4720	return t4_set_params(adap, adap->mbox, adap->pf, 0, 1, &param, &val);
3712	4721	}
3713	4722
3714		-int t4_sge_alloc_uld_txq(struct adapter adap, struct sge_uld_txq txq,
3715		- struct net_device *dev, unsigned int iqid,
3716		- unsigned int uld_type)
	4723	+static int t4_sge_alloc_ofld_txq(struct adapter adap, struct sge_txq q,
	4724	+ struct net_device *dev, u32 cmd, u32 iqid)
3717	4725	{
3718		- int ret, nentries;
3719		- struct fw_eq_ofld_cmd c;
3720		- struct sge *s = &adap->sge;
	4726	+ unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
3721	4727	struct port_info *pi = netdev_priv(dev);
3722		- int cmd = FW_EQ_OFLD_CMD;
	4728	+ struct sge *s = &adap->sge;
	4729	+ struct fw_eq_ofld_cmd c;
	4730	+ u32 fb_min, nentries;
	4731	+ int ret;
3723	4732
3724	4733	/* Add status entries */
3725		- nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
3726		-
3727		- txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size,
3728		- sizeof(struct tx_desc), sizeof(struct tx_sw_desc),
3729		- &txq->q.phys_addr, &txq->q.sdesc, s->stat_len,
3730		- NUMA_NO_NODE);
3731		- if (!txq->q.desc)
	4734	+ nentries = q->size + s->stat_len / sizeof(struct tx_desc);
	4735	+ q->desc = alloc_ring(adap->pdev_dev, q->size, sizeof(struct tx_desc),
	4736	+ sizeof(struct tx_sw_desc), &q->phys_addr,
	4737	+ &q->sdesc, s->stat_len, NUMA_NO_NODE);
	4738	+ if (!q->desc)
3732	4739	return -ENOMEM;
3733	4740
	4741	+ if (chip_ver <= CHELSIO_T5)
	4742	+ fb_min = FETCHBURSTMIN_64B_X;
	4743	+ else
	4744	+ fb_min = FETCHBURSTMIN_64B_T6_X;
	4745	+
3734	4746	memset(&c, 0, sizeof(c));
3735		- if (unlikely(uld_type == CXGB4_TX_CRYPTO))
3736		- cmd = FW_EQ_CTRL_CMD;
3737	4747	c.op_to_vfn = htonl(FW_CMD_OP_V(cmd) \| FW_CMD_REQUEST_F \|
3738	4748	FW_CMD_WRITE_F \| FW_CMD_EXEC_F \|
3739	4749	FW_EQ_OFLD_CMD_PFN_V(adap->pf) \|
..	..	@@ -3745,29 +4755,66 @@
3745	4755	FW_EQ_OFLD_CMD_PCIECHN_V(pi->tx_chan) \|
3746	4756	FW_EQ_OFLD_CMD_FETCHRO_F \| FW_EQ_OFLD_CMD_IQID_V(iqid));
3747	4757	c.dcaen_to_eqsize =
3748		- htonl(FW_EQ_OFLD_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) \|
	4758	+ htonl(FW_EQ_OFLD_CMD_FBMIN_V(fb_min) \|
3749	4759	FW_EQ_OFLD_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) \|
3750	4760	FW_EQ_OFLD_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) \|
3751	4761	FW_EQ_OFLD_CMD_EQSIZE_V(nentries));
3752		- c.eqaddr = cpu_to_be64(txq->q.phys_addr);
	4762	+ c.eqaddr = cpu_to_be64(q->phys_addr);
3753	4763
3754	4764	ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3755	4765	if (ret) {
3756		- kfree(txq->q.sdesc);
3757		- txq->q.sdesc = NULL;
	4766	+ kfree(q->sdesc);
	4767	+ q->sdesc = NULL;
3758	4768	dma_free_coherent(adap->pdev_dev,
3759	4769	nentries * sizeof(struct tx_desc),
3760		- txq->q.desc, txq->q.phys_addr);
3761		- txq->q.desc = NULL;
	4770	+ q->desc, q->phys_addr);
	4771	+ q->desc = NULL;
3762	4772	return ret;
3763	4773	}
3764	4774
	4775	+ init_txq(adap, q, FW_EQ_OFLD_CMD_EQID_G(ntohl(c.eqid_pkd)));
	4776	+ return 0;
	4777	+}
	4778	+
	4779	+int t4_sge_alloc_uld_txq(struct adapter adap, struct sge_uld_txq txq,
	4780	+ struct net_device *dev, unsigned int iqid,
	4781	+ unsigned int uld_type)
	4782	+{
	4783	+ u32 cmd = FW_EQ_OFLD_CMD;
	4784	+ int ret;
	4785	+
	4786	+ if (unlikely(uld_type == CXGB4_TX_CRYPTO))
	4787	+ cmd = FW_EQ_CTRL_CMD;
	4788	+
	4789	+ ret = t4_sge_alloc_ofld_txq(adap, &txq->q, dev, cmd, iqid);
	4790	+ if (ret)
	4791	+ return ret;
	4792	+
3765	4793	txq->q.q_type = CXGB4_TXQ_ULD;
3766		- init_txq(adap, &txq->q, FW_EQ_OFLD_CMD_EQID_G(ntohl(c.eqid_pkd)));
3767	4794	txq->adap = adap;
3768	4795	skb_queue_head_init(&txq->sendq);
3769		- tasklet_init(&txq->qresume_tsk, restart_ofldq, (unsigned long)txq);
	4796	+ tasklet_setup(&txq->qresume_tsk, restart_ofldq);
3770	4797	txq->full = 0;
	4798	+ txq->mapping_err = 0;
	4799	+ return 0;
	4800	+}
	4801	+
	4802	+int t4_sge_alloc_ethofld_txq(struct adapter adap, struct sge_eohw_txq txq,
	4803	+ struct net_device *dev, u32 iqid)
	4804	+{
	4805	+ int ret;
	4806	+
	4807	+ ret = t4_sge_alloc_ofld_txq(adap, &txq->q, dev, FW_EQ_OFLD_CMD, iqid);
	4808	+ if (ret)
	4809	+ return ret;
	4810	+
	4811	+ txq->q.q_type = CXGB4_TXQ_ULD;
	4812	+ spin_lock_init(&txq->lock);
	4813	+ txq->adap = adap;
	4814	+ txq->tso = 0;
	4815	+ txq->uso = 0;
	4816	+ txq->tx_cso = 0;
	4817	+ txq->vlan_ins = 0;
3771	4818	txq->mapping_err = 0;
3772	4819	return 0;
3773	4820	}
..	..	@@ -3827,6 +4874,17 @@
3827	4874	q->fl.size ? &q->fl : NULL);
3828	4875	}
3829	4876
	4877	+void t4_sge_free_ethofld_txq(struct adapter adap, struct sge_eohw_txq txq)
	4878	+{
	4879	+ if (txq->q.desc) {
	4880	+ t4_ofld_eq_free(adap, adap->mbox, adap->pf, 0,
	4881	+ txq->q.cntxt_id);
	4882	+ free_tx_desc(adap, &txq->q, txq->q.in_use, false);
	4883	+ kfree(txq->q.sdesc);
	4884	+ free_txq(adap, &txq->q);
	4885	+ }
	4886	+}
	4887	+
3830	4888	/**
3831	4889	* t4_free_sge_resources - free SGE resources
3832	4890	* @adap: the adapter
..	..	@@ -3856,6 +4914,10 @@
3856	4914	if (eq->rspq.desc)
3857	4915	free_rspq_fl(adap, &eq->rspq,
3858	4916	eq->fl.size ? &eq->fl : NULL);
	4917	+ if (eq->msix) {
	4918	+ cxgb4_free_msix_idx_in_bmap(adap, eq->msix->idx);
	4919	+ eq->msix = NULL;
	4920	+ }
3859	4921
3860	4922	etq = &adap->sge.ethtxq[i];
3861	4923	if (etq->q.desc) {
..	..	@@ -3882,8 +4944,15 @@
3882	4944	}
3883	4945	}
3884	4946
3885		- if (adap->sge.fw_evtq.desc)
	4947	+ if (adap->sge.fw_evtq.desc) {
3886	4948	free_rspq_fl(adap, &adap->sge.fw_evtq, NULL);
	4949	+ if (adap->sge.fwevtq_msix_idx >= 0)
	4950	+ cxgb4_free_msix_idx_in_bmap(adap,
	4951	+ adap->sge.fwevtq_msix_idx);
	4952	+ }
	4953	+
	4954	+ if (adap->sge.nd_msix_idx >= 0)
	4955	+ cxgb4_free_msix_idx_in_bmap(adap, adap->sge.nd_msix_idx);
3887	4956
3888	4957	if (adap->sge.intrq.desc)
3889	4958	free_rspq_fl(adap, &adap->sge.intrq, NULL);
..	..	@@ -3925,9 +4994,6 @@
3925	4994	{
3926	4995	int i;
3927	4996	struct sge *s = &adap->sge;
3928		-
3929		- if (in_interrupt()) /* actions below require waiting */
3930		- return;
3931	4997
3932	4998	if (s->rx_timer.function)
3933	4999	del_timer_sync(&s->rx_timer);