kernel_5.10 no rt

hc

2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f

 kernel/drivers/net/ethernet/sfc/tx.c
..
..
@@ -1,11 +1,8 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /****************************************************************************
2
3
  * Driver for Solarflare network controllers and boards
3
4
  * Copyright 2005-2006 Fen Systems Ltd.
4
5
  * Copyright 2005-2013 Solarflare Communications Inc.
5
- *
6
- * This program is free software; you can redistribute it and/or modify it
7
- * under the terms of the GNU General Public License version 2 as published
8
- * by the Free Software Foundation, incorporated herein by reference.
9
6
  */
10
7
 
11
8
 #include <linux/pci.h>
..
..
@@ -23,6 +20,7 @@
23
20
 #include "io.h"
24
21
 #include "nic.h"
25
22
 #include "tx.h"
23
+#include "tx_common.h"
26
24
 #include "workarounds.h"
27
25
 #include "ef10_regs.h"
28
26
 
..
..
@@ -59,79 +57,14 @@
59
57
 	return efx_tx_get_copy_buffer(tx_queue, buffer);
60
58
 }
61
59
 
62
-static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
63
-			       struct efx_tx_buffer *buffer,
64
-			       unsigned int *pkts_compl,
65
-			       unsigned int *bytes_compl)
66
-{
67
-	if (buffer->unmap_len) {
68
-		struct device *dma_dev = &tx_queue->efx->pci_dev->dev;
69
-		dma_addr_t unmap_addr = buffer->dma_addr - buffer->dma_offset;
70
-		if (buffer->flags & EFX_TX_BUF_MAP_SINGLE)
71
-			dma_unmap_single(dma_dev, unmap_addr, buffer->unmap_len,
72
-					 DMA_TO_DEVICE);
73
-		else
74
-			dma_unmap_page(dma_dev, unmap_addr, buffer->unmap_len,
75
-				       DMA_TO_DEVICE);
76
-		buffer->unmap_len = 0;
77
-	}
78
-
79
-	if (buffer->flags & EFX_TX_BUF_SKB) {
80
-		struct sk_buff *skb = (struct sk_buff *)buffer->skb;
81
-
82
-		EFX_WARN_ON_PARANOID(!pkts_compl || !bytes_compl);
83
-		(*pkts_compl)++;
84
-		(*bytes_compl) += skb->len;
85
-		if (tx_queue->timestamping &&
86
-		    (tx_queue->completed_timestamp_major ||
87
-		     tx_queue->completed_timestamp_minor)) {
88
-			struct skb_shared_hwtstamps hwtstamp;
89
-
90
-			hwtstamp.hwtstamp =
91
-				efx_ptp_nic_to_kernel_time(tx_queue);
92
-			skb_tstamp_tx(skb, &hwtstamp);
93
-
94
-			tx_queue->completed_timestamp_major = 0;
95
-			tx_queue->completed_timestamp_minor = 0;
96
-		}
97
-		dev_consume_skb_any((struct sk_buff *)buffer->skb);
98
-		netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev,
99
-			   "TX queue %d transmission id %x complete\n",
100
-			   tx_queue->queue, tx_queue->read_count);
101
-	}
102
-
103
-	buffer->len = 0;
104
-	buffer->flags = 0;
105
-}
106
-
107
-unsigned int efx_tx_max_skb_descs(struct efx_nic *efx)
108
-{
109
-	/* Header and payload descriptor for each output segment, plus
110
-	 * one for every input fragment boundary within a segment
111
-	 */
112
-	unsigned int max_descs = EFX_TSO_MAX_SEGS * 2 + MAX_SKB_FRAGS;
113
-
114
-	/* Possibly one more per segment for option descriptors */
115
-	if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0)
116
-		max_descs += EFX_TSO_MAX_SEGS;
117
-
118
-	/* Possibly more for PCIe page boundaries within input fragments */
119
-	if (PAGE_SIZE > EFX_PAGE_SIZE)
120
-		max_descs += max_t(unsigned int, MAX_SKB_FRAGS,
121
-				   DIV_ROUND_UP(GSO_MAX_SIZE, EFX_PAGE_SIZE));
122
-
123
-	return max_descs;
124
-}
125
-
126
60
 static void efx_tx_maybe_stop_queue(struct efx_tx_queue *txq1)
127
61
 {
128
-	/* We need to consider both queues that the net core sees as one */
129
-	struct efx_tx_queue *txq2 = efx_tx_queue_partner(txq1);
62
+	/* We need to consider all queues that the net core sees as one */
130
63
 	struct efx_nic *efx = txq1->efx;
64
+	struct efx_tx_queue *txq2;
131
65
 	unsigned int fill_level;
132
66
 
133
-	fill_level = max(txq1->insert_count - txq1->old_read_count,
134
-			 txq2->insert_count - txq2->old_read_count);
67
+	fill_level = efx_channel_tx_old_fill_level(txq1->channel);
135
68
 	if (likely(fill_level < efx->txq_stop_thresh))
136
69
 		return;
137
70
 
..
..
@@ -151,11 +84,10 @@
151
84
 	 */
152
85
 	netif_tx_stop_queue(txq1->core_txq);
153
86
 	smp_mb();
154
-	txq1->old_read_count = READ_ONCE(txq1->read_count);
155
-	txq2->old_read_count = READ_ONCE(txq2->read_count);
87
+	efx_for_each_channel_tx_queue(txq2, txq1->channel)
88
+		txq2->old_read_count = READ_ONCE(txq2->read_count);
156
89
 
157
-	fill_level = max(txq1->insert_count - txq1->old_read_count,
158
-			 txq2->insert_count - txq2->old_read_count);
90
+	fill_level = efx_channel_tx_old_fill_level(txq1->channel);
159
91
 	EFX_WARN_ON_ONCE_PARANOID(fill_level >= efx->txq_entries);
160
92
 	if (likely(fill_level < efx->txq_stop_thresh)) {
161
93
 		smp_mb();
..
..
@@ -277,7 +209,7 @@
277
209
 
278
210
 		vaddr = kmap_atomic(skb_frag_page(f));
279
211
 
280
-		efx_memcpy_toio_aligned_cb(efx, piobuf, vaddr + f->page_offset,
212
+		efx_memcpy_toio_aligned_cb(efx, piobuf, vaddr + skb_frag_off(f),
281
213
 					   skb_frag_size(f), copy_buf);
282
214
 		kunmap_atomic(vaddr);
283
215
 	}
..
..
@@ -332,159 +264,43 @@
332
264
 	++tx_queue->insert_count;
333
265
 	return 0;
334
266
 }
267
+
268
+/* Decide whether we can use TX PIO, ie. write packet data directly into
269
+ * a buffer on the device.  This can reduce latency at the expense of
270
+ * throughput, so we only do this if both hardware and software TX rings
271
+ * are empty, including all queues for the channel.  This also ensures that
272
+ * only one packet at a time can be using the PIO buffer. If the xmit_more
273
+ * flag is set then we don't use this - there'll be another packet along
274
+ * shortly and we want to hold off the doorbell.
275
+ */
276
+static bool efx_tx_may_pio(struct efx_tx_queue *tx_queue)
277
+{
278
+	struct efx_channel *channel = tx_queue->channel;
279
+
280
+	if (!tx_queue->piobuf)
281
+		return false;
282
+
283
+	EFX_WARN_ON_ONCE_PARANOID(!channel->efx->type->option_descriptors);
284
+
285
+	efx_for_each_channel_tx_queue(tx_queue, channel)
286
+		if (!efx_nic_tx_is_empty(tx_queue, tx_queue->packet_write_count))
287
+			return false;
288
+
289
+	return true;
290
+}
335
291
 #endif /* EFX_USE_PIO */
336
292
 
337
-static struct efx_tx_buffer *efx_tx_map_chunk(struct efx_tx_queue *tx_queue,
338
-					      dma_addr_t dma_addr,
339
-					      size_t len)
340
-{
341
-	const struct efx_nic_type *nic_type = tx_queue->efx->type;
342
-	struct efx_tx_buffer *buffer;
343
-	unsigned int dma_len;
344
-
345
-	/* Map the fragment taking account of NIC-dependent DMA limits. */
346
-	do {
347
-		buffer = efx_tx_queue_get_insert_buffer(tx_queue);
348
-		dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len);
349
-
350
-		buffer->len = dma_len;
351
-		buffer->dma_addr = dma_addr;
352
-		buffer->flags = EFX_TX_BUF_CONT;
353
-		len -= dma_len;
354
-		dma_addr += dma_len;
355
-		++tx_queue->insert_count;
356
-	} while (len);
357
-
358
-	return buffer;
359
-}
360
-
361
-/* Map all data from an SKB for DMA and create descriptors on the queue.
293
+/* Send any pending traffic for a channel. xmit_more is shared across all
294
+ * queues for a channel, so we must check all of them.
362
295
  */
363
-static int efx_tx_map_data(struct efx_tx_queue *tx_queue, struct sk_buff *skb,
364
-			   unsigned int segment_count)
296
+static void efx_tx_send_pending(struct efx_channel *channel)
365
297
 {
366
-	struct efx_nic *efx = tx_queue->efx;
367
-	struct device *dma_dev = &efx->pci_dev->dev;
368
-	unsigned int frag_index, nr_frags;
369
-	dma_addr_t dma_addr, unmap_addr;
370
-	unsigned short dma_flags;
371
-	size_t len, unmap_len;
298
+	struct efx_tx_queue *q;
372
299
 
373
-	nr_frags = skb_shinfo(skb)->nr_frags;
374
-	frag_index = 0;
375
-
376
-	/* Map header data. */
377
-	len = skb_headlen(skb);
378
-	dma_addr = dma_map_single(dma_dev, skb->data, len, DMA_TO_DEVICE);
379
-	dma_flags = EFX_TX_BUF_MAP_SINGLE;
380
-	unmap_len = len;
381
-	unmap_addr = dma_addr;
382
-
383
-	if (unlikely(dma_mapping_error(dma_dev, dma_addr)))
384
-		return -EIO;
385
-
386
-	if (segment_count) {
387
-		/* For TSO we need to put the header in to a separate
388
-		 * descriptor. Map this separately if necessary.
389
-		 */
390
-		size_t header_len = skb_transport_header(skb) - skb->data +
391
-				(tcp_hdr(skb)->doff << 2u);
392
-
393
-		if (header_len != len) {
394
-			tx_queue->tso_long_headers++;
395
-			efx_tx_map_chunk(tx_queue, dma_addr, header_len);
396
-			len -= header_len;
397
-			dma_addr += header_len;
398
-		}
300
+	efx_for_each_channel_tx_queue(q, channel) {
301
+		if (q->xmit_pending)
302
+			efx_nic_push_buffers(q);
399
303
 	}
400
-
401
-	/* Add descriptors for each fragment. */
402
-	do {
403
-		struct efx_tx_buffer *buffer;
404
-		skb_frag_t *fragment;
405
-
406
-		buffer = efx_tx_map_chunk(tx_queue, dma_addr, len);
407
-
408
-		/* The final descriptor for a fragment is responsible for
409
-		 * unmapping the whole fragment.
410
-		 */
411
-		buffer->flags = EFX_TX_BUF_CONT | dma_flags;
412
-		buffer->unmap_len = unmap_len;
413
-		buffer->dma_offset = buffer->dma_addr - unmap_addr;
414
-
415
-		if (frag_index >= nr_frags) {
416
-			/* Store SKB details with the final buffer for
417
-			 * the completion.
418
-			 */
419
-			buffer->skb = skb;
420
-			buffer->flags = EFX_TX_BUF_SKB | dma_flags;
421
-			return 0;
422
-		}
423
-
424
-		/* Move on to the next fragment. */
425
-		fragment = &skb_shinfo(skb)->frags[frag_index++];
426
-		len = skb_frag_size(fragment);
427
-		dma_addr = skb_frag_dma_map(dma_dev, fragment,
428
-				0, len, DMA_TO_DEVICE);
429
-		dma_flags = 0;
430
-		unmap_len = len;
431
-		unmap_addr = dma_addr;
432
-
433
-		if (unlikely(dma_mapping_error(dma_dev, dma_addr)))
434
-			return -EIO;
435
-	} while (1);
436
-}
437
-
438
-/* Remove buffers put into a tx_queue for the current packet.
439
- * None of the buffers must have an skb attached.
440
- */
441
-static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue,
442
-			       unsigned int insert_count)
443
-{
444
-	struct efx_tx_buffer *buffer;
445
-	unsigned int bytes_compl = 0;
446
-	unsigned int pkts_compl = 0;
447
-
448
-	/* Work backwards until we hit the original insert pointer value */
449
-	while (tx_queue->insert_count != insert_count) {
450
-		--tx_queue->insert_count;
451
-		buffer = __efx_tx_queue_get_insert_buffer(tx_queue);
452
-		efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl);
453
-	}
454
-}
455
-
456
-/*
457
- * Fallback to software TSO.
458
- *
459
- * This is used if we are unable to send a GSO packet through hardware TSO.
460
- * This should only ever happen due to per-queue restrictions - unsupported
461
- * packets should first be filtered by the feature flags.
462
- *
463
- * Returns 0 on success, error code otherwise.
464
- */
465
-static int efx_tx_tso_fallback(struct efx_tx_queue *tx_queue,
466
-			       struct sk_buff *skb)
467
-{
468
-	struct sk_buff *segments, *next;
469
-
470
-	segments = skb_gso_segment(skb, 0);
471
-	if (IS_ERR(segments))
472
-		return PTR_ERR(segments);
473
-
474
-	dev_kfree_skb_any(skb);
475
-	skb = segments;
476
-
477
-	while (skb) {
478
-		next = skb->next;
479
-		skb->next = NULL;
480
-
481
-		if (next)
482
-			skb->xmit_more = true;
483
-		efx_enqueue_skb(tx_queue, skb);
484
-		skb = next;
485
-	}
486
-
487
-	return 0;
488
304
 }
489
305
 
490
306
 /*
..
..
@@ -503,10 +319,10 @@
503
319
  * Returns NETDEV_TX_OK.
504
320
  * You must hold netif_tx_lock() to call this function.
505
321
  */
506
-netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
322
+netdev_tx_t __efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
507
323
 {
508
324
 	unsigned int old_insert_count = tx_queue->insert_count;
509
-	bool xmit_more = skb->xmit_more;
325
+	bool xmit_more = netdev_xmit_more();
510
326
 	bool data_mapped = false;
511
327
 	unsigned int segments;
512
328
 	unsigned int skb_len;
..
..
@@ -522,8 +338,18 @@
522
338
 	 * size limit.
523
339
 	 */
524
340
 	if (segments) {
525
-		EFX_WARN_ON_ONCE_PARANOID(!tx_queue->handle_tso);
526
-		rc = tx_queue->handle_tso(tx_queue, skb, &data_mapped);
341
+		switch (tx_queue->tso_version) {
342
+		case 1:
343
+			rc = efx_enqueue_skb_tso(tx_queue, skb, &data_mapped);
344
+			break;
345
+		case 2:
346
+			rc = efx_ef10_tx_tso_desc(tx_queue, skb, &data_mapped);
347
+			break;
348
+		case 0: /* No TSO on this queue, SW fallback needed */
349
+		default:
350
+			rc = -EINVAL;
351
+			break;
352
+		}
527
353
 		if (rc == -EINVAL) {
528
354
 			rc = efx_tx_tso_fallback(tx_queue, skb);
529
355
 			tx_queue->tso_fallbacks++;
..
..
@@ -533,8 +359,8 @@
533
359
 		if (rc)
534
360
 			goto err;
535
361
 #ifdef EFX_USE_PIO
536
-	} else if (skb_len <= efx_piobuf_size && !skb->xmit_more &&
537
-		   efx_nic_may_tx_pio(tx_queue)) {
362
+	} else if (skb_len <= efx_piobuf_size && !xmit_more &&
363
+		   efx_tx_may_pio(tx_queue)) {
538
364
 		/* Use PIO for short packets with an empty queue. */
539
365
 		if (efx_enqueue_skb_pio(tx_queue, skb))
540
366
 			goto err;
..
..
@@ -553,26 +379,13 @@
553
379
 	if (!data_mapped && (efx_tx_map_data(tx_queue, skb, segments)))
554
380
 		goto err;
555
381
 
556
-	/* Update BQL */
557
-	netdev_tx_sent_queue(tx_queue->core_txq, skb_len);
558
-
559
382
 	efx_tx_maybe_stop_queue(tx_queue);
560
383
 
384
+	tx_queue->xmit_pending = true;
385
+
561
386
 	/* Pass off to hardware */
562
-	if (!xmit_more || netif_xmit_stopped(tx_queue->core_txq)) {
563
-		struct efx_tx_queue *txq2 = efx_tx_queue_partner(tx_queue);
564
-
565
-		/* There could be packets left on the partner queue if those
566
-		 * SKBs had skb->xmit_more set. If we do not push those they
567
-		 * could be left for a long time and cause a netdev watchdog.
568
-		 */
569
-		if (txq2->xmit_more_available)
570
-			efx_nic_push_buffers(txq2);
571
-
572
-		efx_nic_push_buffers(tx_queue);
573
-	} else {
574
-		tx_queue->xmit_more_available = skb->xmit_more;
575
-	}
387
+	if (__netdev_tx_sent_queue(tx_queue->core_txq, skb_len, xmit_more))
388
+		efx_tx_send_pending(tx_queue->channel);
576
389
 
577
390
 	if (segments) {
578
391
 		tx_queue->tso_bursts++;
..
..
@@ -593,61 +406,105 @@
593
406
 	 * on this queue or a partner queue then we need to push here to get the
594
407
 	 * previous packets out.
595
408
 	 */
596
-	if (!xmit_more) {
597
-		struct efx_tx_queue *txq2 = efx_tx_queue_partner(tx_queue);
598
-
599
-		if (txq2->xmit_more_available)
600
-			efx_nic_push_buffers(txq2);
601
-
602
-		efx_nic_push_buffers(tx_queue);
603
-	}
409
+	if (!xmit_more)
410
+		efx_tx_send_pending(tx_queue->channel);
604
411
 
605
412
 	return NETDEV_TX_OK;
606
413
 }
607
414
 
608
-/* Remove packets from the TX queue
609
- *
610
- * This removes packets from the TX queue, up to and including the
611
- * specified index.
612
- */
613
-static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue,
614
-				unsigned int index,
615
-				unsigned int *pkts_compl,
616
-				unsigned int *bytes_compl)
415
+static void efx_xdp_return_frames(int n,  struct xdp_frame **xdpfs)
617
416
 {
618
-	struct efx_nic *efx = tx_queue->efx;
619
-	unsigned int stop_index, read_ptr;
417
+	int i;
620
418
 
621
-	stop_index = (index + 1) & tx_queue->ptr_mask;
622
-	read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
419
+	for (i = 0; i < n; i++)
420
+		xdp_return_frame_rx_napi(xdpfs[i]);
421
+}
623
422
 
624
-	while (read_ptr != stop_index) {
625
-		struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];
423
+/* Transmit a packet from an XDP buffer
424
+ *
425
+ * Returns number of packets sent on success, error code otherwise.
426
+ * Runs in NAPI context, either in our poll (for XDP TX) or a different NIC
427
+ * (for XDP redirect).
428
+ */
429
+int efx_xdp_tx_buffers(struct efx_nic *efx, int n, struct xdp_frame **xdpfs,
430
+		       bool flush)
431
+{
432
+	struct efx_tx_buffer *tx_buffer;
433
+	struct efx_tx_queue *tx_queue;
434
+	struct xdp_frame *xdpf;
435
+	dma_addr_t dma_addr;
436
+	unsigned int len;
437
+	int space;
438
+	int cpu;
439
+	int i;
626
440
 
627
-		if (!(buffer->flags & EFX_TX_BUF_OPTION) &&
628
-		    unlikely(buffer->len == 0)) {
629
-			netif_err(efx, tx_err, efx->net_dev,
630
-				  "TX queue %d spurious TX completion id %x\n",
631
-				  tx_queue->queue, read_ptr);
632
-			efx_schedule_reset(efx, RESET_TYPE_TX_SKIP);
633
-			return;
634
-		}
441
+	cpu = raw_smp_processor_id();
635
442
 
636
-		efx_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl);
443
+	if (!efx->xdp_tx_queue_count ||
444
+	    unlikely(cpu >= efx->xdp_tx_queue_count))
445
+		return -EINVAL;
637
446
 
638
-		++tx_queue->read_count;
639
-		read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
447
+	tx_queue = efx->xdp_tx_queues[cpu];
448
+	if (unlikely(!tx_queue))
449
+		return -EINVAL;
450
+
451
+	if (unlikely(n && !xdpfs))
452
+		return -EINVAL;
453
+
454
+	if (!n)
455
+		return 0;
456
+
457
+	/* Check for available space. We should never need multiple
458
+	 * descriptors per frame.
459
+	 */
460
+	space = efx->txq_entries +
461
+		tx_queue->read_count - tx_queue->insert_count;
462
+
463
+	for (i = 0; i < n; i++) {
464
+		xdpf = xdpfs[i];
465
+
466
+		if (i >= space)
467
+			break;
468
+
469
+		/* We'll want a descriptor for this tx. */
470
+		prefetchw(__efx_tx_queue_get_insert_buffer(tx_queue));
471
+
472
+		len = xdpf->len;
473
+
474
+		/* Map for DMA. */
475
+		dma_addr = dma_map_single(&efx->pci_dev->dev,
476
+					  xdpf->data, len,
477
+					  DMA_TO_DEVICE);
478
+		if (dma_mapping_error(&efx->pci_dev->dev, dma_addr))
479
+			break;
480
+
481
+		/*  Create descriptor and set up for unmapping DMA. */
482
+		tx_buffer = efx_tx_map_chunk(tx_queue, dma_addr, len);
483
+		tx_buffer->xdpf = xdpf;
484
+		tx_buffer->flags = EFX_TX_BUF_XDP |
485
+				   EFX_TX_BUF_MAP_SINGLE;
486
+		tx_buffer->dma_offset = 0;
487
+		tx_buffer->unmap_len = len;
488
+		tx_queue->tx_packets++;
640
489
 	}
490
+
491
+	/* Pass mapped frames to hardware. */
492
+	if (flush && i > 0)
493
+		efx_nic_push_buffers(tx_queue);
494
+
495
+	if (i == 0)
496
+		return -EIO;
497
+
498
+	efx_xdp_return_frames(n - i, xdpfs + i);
499
+
500
+	return i;
641
501
 }
642
502
 
643
503
 /* Initiate a packet transmission.  We use one channel per CPU
644
- * (sharing when we have more CPUs than channels).  On Falcon, the TX
645
- * completion events will be directed back to the CPU that transmitted
646
- * the packet, which should be cache-efficient.
504
+ * (sharing when we have more CPUs than channels).
647
505
  *
648
506
  * Context: non-blocking.
649
- * Note that returning anything other than NETDEV_TX_OK will cause the
650
- * OS to free the skb.
507
+ * Should always return NETDEV_TX_OK and consume the skb.
651
508
  */
652
509
 netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb,
653
510
 				struct net_device *net_dev)
..
..
@@ -658,21 +515,79 @@
658
515
 
659
516
 	EFX_WARN_ON_PARANOID(!netif_device_present(net_dev));
660
517
 
661
-	/* PTP "event" packet */
662
-	if (unlikely(efx_xmit_with_hwtstamp(skb)) &&
663
-	    unlikely(efx_ptp_is_ptp_tx(efx, skb))) {
664
-		return efx_ptp_tx(efx, skb);
665
-	}
666
-
667
518
 	index = skb_get_queue_mapping(skb);
668
-	type = skb->ip_summed == CHECKSUM_PARTIAL ? EFX_TXQ_TYPE_OFFLOAD : 0;
519
+	type = efx_tx_csum_type_skb(skb);
669
520
 	if (index >= efx->n_tx_channels) {
670
521
 		index -= efx->n_tx_channels;
671
522
 		type |= EFX_TXQ_TYPE_HIGHPRI;
672
523
 	}
673
-	tx_queue = efx_get_tx_queue(efx, index, type);
674
524
 
675
-	return efx_enqueue_skb(tx_queue, skb);
525
+	/* PTP "event" packet */
526
+	if (unlikely(efx_xmit_with_hwtstamp(skb)) &&
527
+	    unlikely(efx_ptp_is_ptp_tx(efx, skb))) {
528
+		/* There may be existing transmits on the channel that are
529
+		 * waiting for this packet to trigger the doorbell write.
530
+		 * We need to send the packets at this point.
531
+		 */
532
+		efx_tx_send_pending(efx_get_tx_channel(efx, index));
533
+		return efx_ptp_tx(efx, skb);
534
+	}
535
+
536
+	tx_queue = efx_get_tx_queue(efx, index, type);
537
+	if (WARN_ON_ONCE(!tx_queue)) {
538
+		/* We don't have a TXQ of the right type.
539
+		 * This should never happen, as we don't advertise offload
540
+		 * features unless we can support them.
541
+		 */
542
+		dev_kfree_skb_any(skb);
543
+		/* If we're not expecting another transmit and we had something to push
544
+		 * on this queue or a partner queue then we need to push here to get the
545
+		 * previous packets out.
546
+		 */
547
+		if (!netdev_xmit_more())
548
+			efx_tx_send_pending(efx_get_tx_channel(efx, index));
549
+		return NETDEV_TX_OK;
550
+	}
551
+
552
+	return __efx_enqueue_skb(tx_queue, skb);
553
+}
554
+
555
+void efx_xmit_done_single(struct efx_tx_queue *tx_queue)
556
+{
557
+	unsigned int pkts_compl = 0, bytes_compl = 0;
558
+	unsigned int read_ptr;
559
+	bool finished = false;
560
+
561
+	read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
562
+
563
+	while (!finished) {
564
+		struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];
565
+
566
+		if (!efx_tx_buffer_in_use(buffer)) {
567
+			struct efx_nic *efx = tx_queue->efx;
568
+
569
+			netif_err(efx, hw, efx->net_dev,
570
+				  "TX queue %d spurious single TX completion\n",
571
+				  tx_queue->queue);
572
+			efx_schedule_reset(efx, RESET_TYPE_TX_SKIP);
573
+			return;
574
+		}
575
+
576
+		/* Need to check the flag before dequeueing. */
577
+		if (buffer->flags & EFX_TX_BUF_SKB)
578
+			finished = true;
579
+		efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl);
580
+
581
+		++tx_queue->read_count;
582
+		read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
583
+	}
584
+
585
+	tx_queue->pkts_compl += pkts_compl;
586
+	tx_queue->bytes_compl += bytes_compl;
587
+
588
+	EFX_WARN_ON_PARANOID(pkts_compl != 1);
589
+
590
+	efx_xmit_done_check_empty(tx_queue);
676
591
 }
677
592
 
678
593
 void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue)
..
..
@@ -682,8 +597,8 @@
682
597
 	/* Must be inverse of queue lookup in efx_hard_start_xmit() */
683
598
 	tx_queue->core_txq =
684
599
 		netdev_get_tx_queue(efx->net_dev,
685
-				    tx_queue->queue / EFX_TXQ_TYPES +
686
-				    ((tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ?
600
+				    tx_queue->channel->channel +
601
+				    ((tx_queue->type & EFX_TXQ_TYPE_HIGHPRI) ?
687
602
 				     efx->n_tx_channels : 0));
688
603
 }
689
604
 
..
..
@@ -692,12 +607,13 @@
692
607
 {
693
608
 	struct efx_nic *efx = netdev_priv(net_dev);
694
609
 	struct tc_mqprio_qopt *mqprio = type_data;
695
-	struct efx_channel *channel;
696
-	struct efx_tx_queue *tx_queue;
697
610
 	unsigned tc, num_tc;
698
-	int rc;
699
611
 
700
612
 	if (type != TC_SETUP_QDISC_MQPRIO)
613
+		return -EOPNOTSUPP;
614
+
615
+	/* Only Siena supported highpri queues */
616
+	if (efx_nic_rev(efx) > EFX_REV_SIENA_A0)
701
617
 		return -EOPNOTSUPP;
702
618
 
703
619
 	num_tc = mqprio->num_tc;
..
..
@@ -715,208 +631,9 @@
715
631
 		net_dev->tc_to_txq[tc].count = efx->n_tx_channels;
716
632
 	}
717
633
 
718
-	if (num_tc > net_dev->num_tc) {
719
-		/* Initialise high-priority queues as necessary */
720
-		efx_for_each_channel(channel, efx) {
721
-			efx_for_each_possible_channel_tx_queue(tx_queue,
722
-							       channel) {
723
-				if (!(tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI))
724
-					continue;
725
-				if (!tx_queue->buffer) {
726
-					rc = efx_probe_tx_queue(tx_queue);
727
-					if (rc)
728
-						return rc;
729
-				}
730
-				if (!tx_queue->initialised)
731
-					efx_init_tx_queue(tx_queue);
732
-				efx_init_tx_queue_core_txq(tx_queue);
733
-			}
734
-		}
735
-	} else {
736
-		/* Reduce number of classes before number of queues */
737
-		net_dev->num_tc = num_tc;
738
-	}
739
-
740
-	rc = netif_set_real_num_tx_queues(net_dev,
741
-					  max_t(int, num_tc, 1) *
742
-					  efx->n_tx_channels);
743
-	if (rc)
744
-		return rc;
745
-
746
-	/* Do not destroy high-priority queues when they become
747
-	 * unused.  We would have to flush them first, and it is
748
-	 * fairly difficult to flush a subset of TX queues.  Leave
749
-	 * it to efx_fini_channels().
750
-	 */
751
-
752
634
 	net_dev->num_tc = num_tc;
753
-	return 0;
754
-}
755
635
 
756
-void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)
757
-{
758
-	unsigned fill_level;
759
-	struct efx_nic *efx = tx_queue->efx;
760
-	struct efx_tx_queue *txq2;
761
-	unsigned int pkts_compl = 0, bytes_compl = 0;
762
-
763
-	EFX_WARN_ON_ONCE_PARANOID(index > tx_queue->ptr_mask);
764
-
765
-	efx_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl);
766
-	tx_queue->pkts_compl += pkts_compl;
767
-	tx_queue->bytes_compl += bytes_compl;
768
-
769
-	if (pkts_compl > 1)
770
-		++tx_queue->merge_events;
771
-
772
-	/* See if we need to restart the netif queue.  This memory
773
-	 * barrier ensures that we write read_count (inside
774
-	 * efx_dequeue_buffers()) before reading the queue status.
775
-	 */
776
-	smp_mb();
777
-	if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) &&
778
-	    likely(efx->port_enabled) &&
779
-	    likely(netif_device_present(efx->net_dev))) {
780
-		txq2 = efx_tx_queue_partner(tx_queue);
781
-		fill_level = max(tx_queue->insert_count - tx_queue->read_count,
782
-				 txq2->insert_count - txq2->read_count);
783
-		if (fill_level <= efx->txq_wake_thresh)
784
-			netif_tx_wake_queue(tx_queue->core_txq);
785
-	}
786
-
787
-	/* Check whether the hardware queue is now empty */
788
-	if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) {
789
-		tx_queue->old_write_count = READ_ONCE(tx_queue->write_count);
790
-		if (tx_queue->read_count == tx_queue->old_write_count) {
791
-			smp_mb();
792
-			tx_queue->empty_read_count =
793
-				tx_queue->read_count | EFX_EMPTY_COUNT_VALID;
794
-		}
795
-	}
796
-}
797
-
798
-static unsigned int efx_tx_cb_page_count(struct efx_tx_queue *tx_queue)
799
-{
800
-	return DIV_ROUND_UP(tx_queue->ptr_mask + 1, PAGE_SIZE >> EFX_TX_CB_ORDER);
801
-}
802
-
803
-int efx_probe_tx_queue(struct efx_tx_queue *tx_queue)
804
-{
805
-	struct efx_nic *efx = tx_queue->efx;
806
-	unsigned int entries;
807
-	int rc;
808
-
809
-	/* Create the smallest power-of-two aligned ring */
810
-	entries = max(roundup_pow_of_two(efx->txq_entries), EFX_MIN_DMAQ_SIZE);
811
-	EFX_WARN_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
812
-	tx_queue->ptr_mask = entries - 1;
813
-
814
-	netif_dbg(efx, probe, efx->net_dev,
815
-		  "creating TX queue %d size %#x mask %#x\n",
816
-		  tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask);
817
-
818
-	/* Allocate software ring */
819
-	tx_queue->buffer = kcalloc(entries, sizeof(*tx_queue->buffer),
820
-				   GFP_KERNEL);
821
-	if (!tx_queue->buffer)
822
-		return -ENOMEM;
823
-
824
-	tx_queue->cb_page = kcalloc(efx_tx_cb_page_count(tx_queue),
825
-				    sizeof(tx_queue->cb_page[0]), GFP_KERNEL);
826
-	if (!tx_queue->cb_page) {
827
-		rc = -ENOMEM;
828
-		goto fail1;
829
-	}
830
-
831
-	/* Allocate hardware ring */
832
-	rc = efx_nic_probe_tx(tx_queue);
833
-	if (rc)
834
-		goto fail2;
835
-
836
-	return 0;
837
-
838
-fail2:
839
-	kfree(tx_queue->cb_page);
840
-	tx_queue->cb_page = NULL;
841
-fail1:
842
-	kfree(tx_queue->buffer);
843
-	tx_queue->buffer = NULL;
844
-	return rc;
845
-}
846
-
847
-void efx_init_tx_queue(struct efx_tx_queue *tx_queue)
848
-{
849
-	struct efx_nic *efx = tx_queue->efx;
850
-
851
-	netif_dbg(efx, drv, efx->net_dev,
852
-		  "initialising TX queue %d\n", tx_queue->queue);
853
-
854
-	tx_queue->insert_count = 0;
855
-	tx_queue->write_count = 0;
856
-	tx_queue->packet_write_count = 0;
857
-	tx_queue->old_write_count = 0;
858
-	tx_queue->read_count = 0;
859
-	tx_queue->old_read_count = 0;
860
-	tx_queue->empty_read_count = 0 | EFX_EMPTY_COUNT_VALID;
861
-	tx_queue->xmit_more_available = false;
862
-	tx_queue->timestamping = (efx_ptp_use_mac_tx_timestamps(efx) &&
863
-				  tx_queue->channel == efx_ptp_channel(efx));
864
-	tx_queue->completed_desc_ptr = tx_queue->ptr_mask;
865
-	tx_queue->completed_timestamp_major = 0;
866
-	tx_queue->completed_timestamp_minor = 0;
867
-
868
-	/* Set up default function pointers. These may get replaced by
869
-	 * efx_nic_init_tx() based off NIC/queue capabilities.
870
-	 */
871
-	tx_queue->handle_tso = efx_enqueue_skb_tso;
872
-
873
-	/* Set up TX descriptor ring */
874
-	efx_nic_init_tx(tx_queue);
875
-
876
-	tx_queue->initialised = true;
877
-}
878
-
879
-void efx_fini_tx_queue(struct efx_tx_queue *tx_queue)
880
-{
881
-	struct efx_tx_buffer *buffer;
882
-
883
-	netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
884
-		  "shutting down TX queue %d\n", tx_queue->queue);
885
-
886
-	if (!tx_queue->buffer)
887
-		return;
888
-
889
-	/* Free any buffers left in the ring */
890
-	while (tx_queue->read_count != tx_queue->write_count) {
891
-		unsigned int pkts_compl = 0, bytes_compl = 0;
892
-		buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask];
893
-		efx_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl);
894
-
895
-		++tx_queue->read_count;
896
-	}
897
-	tx_queue->xmit_more_available = false;
898
-	netdev_tx_reset_queue(tx_queue->core_txq);
899
-}
900
-
901
-void efx_remove_tx_queue(struct efx_tx_queue *tx_queue)
902
-{
903
-	int i;
904
-
905
-	if (!tx_queue->buffer)
906
-		return;
907
-
908
-	netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
909
-		  "destroying TX queue %d\n", tx_queue->queue);
910
-	efx_nic_remove_tx(tx_queue);
911
-
912
-	if (tx_queue->cb_page) {
913
-		for (i = 0; i < efx_tx_cb_page_count(tx_queue); i++)
914
-			efx_nic_free_buffer(tx_queue->efx,
915
-					    &tx_queue->cb_page[i]);
916
-		kfree(tx_queue->cb_page);
917
-		tx_queue->cb_page = NULL;
918
-	}
919
-
920
-	kfree(tx_queue->buffer);
921
-	tx_queue->buffer = NULL;
636
+	return netif_set_real_num_tx_queues(net_dev,
637
+					    max_t(int, num_tc, 1) *
638
+					    efx->n_tx_channels);
922
639
 }