forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/drivers/net/ethernet/huawei/hinic/hinic_rx.c
....@@ -1,16 +1,7 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * Huawei HiNIC PCI Express Linux driver
34  * Copyright(c) 2017 Huawei Technologies Co., Ltd
4
- *
5
- * This program is free software; you can redistribute it and/or modify it
6
- * under the terms and conditions of the GNU General Public License,
7
- * version 2, as published by the Free Software Foundation.
8
- *
9
- * This program is distributed in the hope it will be useful, but WITHOUT
10
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12
- * for more details.
13
- *
145  */
156 
167 #include <linux/kernel.h>
....@@ -27,6 +18,7 @@
2718 #include <linux/dma-mapping.h>
2819 #include <linux/prefetch.h>
2920 #include <linux/cpumask.h>
21
+#include <linux/if_vlan.h>
3022 #include <asm/barrier.h>
3123 
3224 #include "hinic_common.h"
....@@ -43,6 +35,16 @@
4335 #define RX_IRQ_NO_LLI_TIMER             0
4436 #define RX_IRQ_NO_CREDIT                0
4537 #define RX_IRQ_NO_RESEND_TIMER          0
38
+#define HINIC_RX_BUFFER_WRITE           16
39
+
40
+#define HINIC_RX_IPV6_PKT		7
41
+#define LRO_PKT_HDR_LEN_IPV4		66
42
+#define LRO_PKT_HDR_LEN_IPV6		86
43
+#define LRO_REPLENISH_THLD		256
44
+
45
+#define LRO_PKT_HDR_LEN(cqe)		\
46
+	(HINIC_GET_RX_PKT_TYPE(be32_to_cpu((cqe)->offload_type)) == \
47
+	 HINIC_RX_IPV6_PKT ? LRO_PKT_HDR_LEN_IPV6 : LRO_PKT_HDR_LEN_IPV4)
4648 
4749 /**
4850  * hinic_rxq_clean_stats - Clean the statistics of specific queue
....@@ -55,6 +57,9 @@
5557 	u64_stats_update_begin(&rxq_stats->syncp);
5658 	rxq_stats->pkts  = 0;
5759 	rxq_stats->bytes = 0;
60
+	rxq_stats->errors = 0;
61
+	rxq_stats->csum_errors = 0;
62
+	rxq_stats->other_errors = 0;
5863 	u64_stats_update_end(&rxq_stats->syncp);
5964 }
6065 
....@@ -68,13 +73,15 @@
6873 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
6974 	unsigned int start;
7075 
71
-	u64_stats_update_begin(&stats->syncp);
7276 	do {
73
-		start = u64_stats_fetch_begin(&rxq_stats->syncp);
77
+		start = u64_stats_fetch_begin_irq(&rxq_stats->syncp);
7478 		stats->pkts = rxq_stats->pkts;
7579 		stats->bytes = rxq_stats->bytes;
76
-	} while (u64_stats_fetch_retry(&rxq_stats->syncp, start));
77
-	u64_stats_update_end(&stats->syncp);
80
+		stats->errors = rxq_stats->csum_errors +
81
+				rxq_stats->other_errors;
82
+		stats->csum_errors = rxq_stats->csum_errors;
83
+		stats->other_errors = rxq_stats->other_errors;
84
+	} while (u64_stats_fetch_retry_irq(&rxq_stats->syncp, start));
7885 }
7986 
8087 /**
....@@ -89,6 +96,26 @@
8996 	hinic_rxq_clean_stats(rxq);
9097 }
9198 
99
+static void rx_csum(struct hinic_rxq *rxq, u32 status,
100
+		    struct sk_buff *skb)
101
+{
102
+	struct net_device *netdev = rxq->netdev;
103
+	u32 csum_err;
104
+
105
+	csum_err = HINIC_RQ_CQE_STATUS_GET(status, CSUM_ERR);
106
+
107
+	if (!(netdev->features & NETIF_F_RXCSUM))
108
+		return;
109
+
110
+	if (!csum_err) {
111
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
112
+	} else {
113
+		if (!(csum_err & (HINIC_RX_CSUM_HW_CHECK_NONE |
114
+			HINIC_RX_CSUM_IPSU_OTHER_ERR)))
115
+			rxq->rxq_stats.csum_errors++;
116
+		skb->ip_summed = CHECKSUM_NONE;
117
+	}
118
+}
92119 /**
93120  * rx_alloc_skb - allocate skb and map it to dma address
94121  * @rxq: rx queue
....@@ -207,7 +234,6 @@
207234 		wmb();  /* write all the wqes before update PI */
208235 
209236 		hinic_rq_update(rxq->rq, prod_idx);
210
-		tasklet_schedule(&rxq->rx_task);
211237 	}
212238 
213239 	return i;
....@@ -234,17 +260,6 @@
234260 
235261 		rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge));
236262 	}
237
-}
238
-
239
-/**
240
- * rx_alloc_task - tasklet for queue allocation
241
- * @data: rx queue
242
- **/
243
-static void rx_alloc_task(unsigned long data)
244
-{
245
-	struct hinic_rxq *rxq = (struct hinic_rxq *)data;
246
-
247
-	(void)rx_alloc_pkts(rxq);
248263 }
249264 
250265 /**
....@@ -299,6 +314,39 @@
299314 	return num_wqes;
300315 }
301316 
317
+static void hinic_copy_lp_data(struct hinic_dev *nic_dev,
318
+			       struct sk_buff *skb)
319
+{
320
+	struct net_device *netdev = nic_dev->netdev;
321
+	u8 *lb_buf = nic_dev->lb_test_rx_buf;
322
+	int lb_len = nic_dev->lb_pkt_len;
323
+	int pkt_offset, frag_len, i;
324
+	void *frag_data = NULL;
325
+
326
+	if (nic_dev->lb_test_rx_idx == LP_PKT_CNT) {
327
+		nic_dev->lb_test_rx_idx = 0;
328
+		netif_warn(nic_dev, drv, netdev, "Loopback test warning, receive too more test pkts\n");
329
+	}
330
+
331
+	if (skb->len != nic_dev->lb_pkt_len) {
332
+		netif_warn(nic_dev, drv, netdev, "Wrong packet length\n");
333
+		nic_dev->lb_test_rx_idx++;
334
+		return;
335
+	}
336
+
337
+	pkt_offset = nic_dev->lb_test_rx_idx * lb_len;
338
+	frag_len = (int)skb_headlen(skb);
339
+	memcpy(lb_buf + pkt_offset, skb->data, frag_len);
340
+	pkt_offset += frag_len;
341
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
342
+		frag_data = skb_frag_address(&skb_shinfo(skb)->frags[i]);
343
+		frag_len = (int)skb_frag_size(&skb_shinfo(skb)->frags[i]);
344
+		memcpy((lb_buf + pkt_offset), frag_data, frag_len);
345
+		pkt_offset += frag_len;
346
+	}
347
+	nic_dev->lb_test_rx_idx++;
348
+}
349
+
302350 /**
303351  * rxq_recv - Rx handler
304352  * @rxq: rx queue
....@@ -309,12 +357,24 @@
309357 static int rxq_recv(struct hinic_rxq *rxq, int budget)
310358 {
311359 	struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq);
360
+	struct net_device *netdev = rxq->netdev;
312361 	u64 pkt_len = 0, rx_bytes = 0;
362
+	struct hinic_rq *rq = rxq->rq;
313363 	struct hinic_rq_wqe *rq_wqe;
364
+	struct hinic_dev *nic_dev;
365
+	unsigned int free_wqebbs;
366
+	struct hinic_rq_cqe *cqe;
314367 	int num_wqes, pkts = 0;
315368 	struct hinic_sge sge;
369
+	unsigned int status;
316370 	struct sk_buff *skb;
317
-	u16 ci;
371
+	u32 offload_type;
372
+	u16 ci, num_lro;
373
+	u16 num_wqe = 0;
374
+	u32 vlan_len;
375
+	u16 vid;
376
+
377
+	nic_dev = netdev_priv(netdev);
318378 
319379 	while (pkts < budget) {
320380 		num_wqes = 0;
....@@ -324,9 +384,16 @@
324384 		if (!rq_wqe)
325385 			break;
326386 
387
+		/* make sure we read rx_done before packet length */
388
+		dma_rmb();
389
+
390
+		cqe = rq->cqe[ci];
391
+		status =  be32_to_cpu(cqe->status);
327392 		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
328393 
329394 		rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
395
+
396
+		rx_csum(rxq, status, skb);
330397 
331398 		prefetch(skb->data);
332399 
....@@ -340,8 +407,19 @@
340407 						     HINIC_RX_BUF_SZ, ci);
341408 		}
342409 
343
-		hinic_rq_put_wqe(rxq->rq, ci,
410
+		hinic_rq_put_wqe(rq, ci,
344411 				 (num_wqes + 1) * HINIC_RQ_WQE_SIZE);
412
+
413
+		offload_type = be32_to_cpu(cqe->offload_type);
414
+		vlan_len = be32_to_cpu(cqe->len);
415
+		if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
416
+		    HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type)) {
417
+			vid = HINIC_GET_RX_VLAN_TAG(vlan_len);
418
+			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
419
+		}
420
+
421
+		if (unlikely(nic_dev->flags & HINIC_LP_TEST))
422
+			hinic_copy_lp_data(nic_dev, skb);
345423 
346424 		skb_record_rx_queue(skb, qp->q_id);
347425 		skb->protocol = eth_type_trans(skb, rxq->netdev);
....@@ -350,10 +428,26 @@
350428 
351429 		pkts++;
352430 		rx_bytes += pkt_len;
431
+
432
+		num_lro = HINIC_GET_RX_NUM_LRO(status);
433
+		if (num_lro) {
434
+			rx_bytes += ((num_lro - 1) *
435
+				     LRO_PKT_HDR_LEN(cqe));
436
+
437
+			num_wqe +=
438
+			(u16)(pkt_len >> rxq->rx_buff_shift) +
439
+			((pkt_len & (rxq->buf_len - 1)) ? 1 : 0);
440
+		}
441
+
442
+		cqe->status = 0;
443
+
444
+		if (num_wqe >= LRO_REPLENISH_THLD)
445
+			break;
353446 	}
354447 
355
-	if (pkts)
356
-		tasklet_schedule(&rxq->rx_task); /* rx_alloc_pkts */
448
+	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);
449
+	if (free_wqebbs > HINIC_RX_BUFFER_WRITE)
450
+		rx_alloc_pkts(rxq);
357451 
358452 	u64_stats_update_begin(&rxq->rxq_stats.syncp);
359453 	rxq->rxq_stats.pkts += pkts;
....@@ -366,6 +460,7 @@
366460 static int rx_poll(struct napi_struct *napi, int budget)
367461 {
368462 	struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi);
463
+	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
369464 	struct hinic_rq *rq = rxq->rq;
370465 	int pkts;
371466 
....@@ -374,7 +469,12 @@
374469 		return budget;
375470 
376471 	napi_complete(napi);
377
-	enable_irq(rq->irq);
472
+
473
+	if (!HINIC_IS_VF(nic_dev->hwdev->hwif))
474
+		hinic_hwdev_set_msix_state(nic_dev->hwdev,
475
+					   rq->msix_entry,
476
+					   HINIC_MSIX_ENABLE);
477
+
378478 	return pkts;
379479 }
380480 
....@@ -399,7 +499,11 @@
399499 	struct hinic_dev *nic_dev;
400500 
401501 	/* Disable the interrupt until napi will be completed */
402
-	disable_irq_nosync(rq->irq);
502
+	nic_dev = netdev_priv(rxq->netdev);
503
+	if (!HINIC_IS_VF(nic_dev->hwdev->hwif))
504
+		hinic_hwdev_set_msix_state(nic_dev->hwdev,
505
+					   rq->msix_entry,
506
+					   HINIC_MSIX_DISABLE);
403507 
404508 	nic_dev = netdev_priv(rxq->netdev);
405509 	hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry);
....@@ -411,10 +515,14 @@
411515 static int rx_request_irq(struct hinic_rxq *rxq)
412516 {
413517 	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
518
+	struct hinic_msix_config interrupt_info = {0};
519
+	struct hinic_intr_coal_info *intr_coal = NULL;
414520 	struct hinic_hwdev *hwdev = nic_dev->hwdev;
415521 	struct hinic_rq *rq = rxq->rq;
416522 	struct hinic_qp *qp;
417523 	int err;
524
+
525
+	qp = container_of(rq, struct hinic_qp, rq);
418526 
419527 	rx_add_napi(rxq);
420528 
....@@ -423,15 +531,35 @@
423531 			     RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT,
424532 			     RX_IRQ_NO_RESEND_TIMER);
425533 
426
-	err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);
534
+	intr_coal = &nic_dev->rx_intr_coalesce[qp->q_id];
535
+	interrupt_info.msix_index = rq->msix_entry;
536
+	interrupt_info.coalesce_timer_cnt = intr_coal->coalesce_timer_cfg;
537
+	interrupt_info.pending_cnt = intr_coal->pending_limt;
538
+	interrupt_info.resend_timer_cnt = intr_coal->resend_timer_cfg;
539
+
540
+	err = hinic_set_interrupt_cfg(hwdev, &interrupt_info);
427541 	if (err) {
428
-		rx_del_napi(rxq);
429
-		return err;
542
+		netif_err(nic_dev, drv, rxq->netdev,
543
+			  "Failed to set RX interrupt coalescing attribute\n");
544
+		goto err_req_irq;
430545 	}
431546 
432
-	qp = container_of(rq, struct hinic_qp, rq);
547
+	err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);
548
+	if (err)
549
+		goto err_req_irq;
550
+
433551 	cpumask_set_cpu(qp->q_id % num_online_cpus(), &rq->affinity_mask);
434
-	return irq_set_affinity_hint(rq->irq, &rq->affinity_mask);
552
+	err = irq_set_affinity_hint(rq->irq, &rq->affinity_mask);
553
+	if (err)
554
+		goto err_irq_affinity;
555
+
556
+	return 0;
557
+
558
+err_irq_affinity:
559
+	free_irq(rq->irq, rxq);
560
+err_req_irq:
561
+	rx_del_napi(rxq);
562
+	return err;
435563 }
436564 
437565 static void rx_free_irq(struct hinic_rxq *rxq)
....@@ -455,21 +583,19 @@
455583 		   struct net_device *netdev)
456584 {
457585 	struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq);
458
-	int err, pkts, irqname_len;
586
+	int err, pkts;
459587 
460588 	rxq->netdev = netdev;
461589 	rxq->rq = rq;
590
+	rxq->buf_len = HINIC_RX_BUF_SZ;
591
+	rxq->rx_buff_shift = ilog2(HINIC_RX_BUF_SZ);
462592 
463593 	rxq_stats_init(rxq);
464594 
465
-	irqname_len = snprintf(NULL, 0, "hinic_rxq%d", qp->q_id) + 1;
466
-	rxq->irq_name = devm_kzalloc(&netdev->dev, irqname_len, GFP_KERNEL);
595
+	rxq->irq_name = devm_kasprintf(&netdev->dev, GFP_KERNEL,
596
+				       "%s_rxq%d", netdev->name, qp->q_id);
467597 	if (!rxq->irq_name)
468598 		return -ENOMEM;
469
-
470
-	sprintf(rxq->irq_name, "hinic_rxq%d", qp->q_id);
471
-
472
-	tasklet_init(&rxq->rx_task, rx_alloc_task, (unsigned long)rxq);
473599 
474600 	pkts = rx_alloc_pkts(rxq);
475601 	if (!pkts) {
....@@ -487,7 +613,6 @@
487613 
488614 err_req_rx_irq:
489615 err_rx_pkts:
490
-	tasklet_kill(&rxq->rx_task);
491616 	free_all_rx_skbs(rxq);
492617 	devm_kfree(&netdev->dev, rxq->irq_name);
493618 	return err;
....@@ -503,7 +628,6 @@
503628 
504629 	rx_free_irq(rxq);
505630 
506
-	tasklet_kill(&rxq->rx_task);
507631 	free_all_rx_skbs(rxq);
508632 	devm_kfree(&netdev->dev, rxq->irq_name);
509633 }