.. | .. |
---|
43 | 43 | #include <linux/ip.h> |
---|
44 | 44 | #include <linux/ipv6.h> |
---|
45 | 45 | #include <linux/moduleparam.h> |
---|
| 46 | +#include <linux/indirect_call_wrapper.h> |
---|
46 | 47 | |
---|
47 | 48 | #include "mlx4_en.h" |
---|
48 | 49 | |
---|
.. | .. |
---|
57 | 58 | |
---|
58 | 59 | ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node); |
---|
59 | 60 | if (!ring) { |
---|
60 | | - ring = kzalloc(sizeof(*ring), GFP_KERNEL); |
---|
61 | | - if (!ring) { |
---|
62 | | - en_err(priv, "Failed allocating TX ring\n"); |
---|
63 | | - return -ENOMEM; |
---|
64 | | - } |
---|
| 61 | + en_err(priv, "Failed allocating TX ring\n"); |
---|
| 62 | + return -ENOMEM; |
---|
65 | 63 | } |
---|
66 | 64 | |
---|
67 | 65 | ring->size = size; |
---|
.. | .. |
---|
264 | 262 | } |
---|
265 | 263 | } |
---|
266 | 264 | |
---|
| 265 | +INDIRECT_CALLABLE_DECLARE(u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, |
---|
| 266 | + struct mlx4_en_tx_ring *ring, |
---|
| 267 | + int index, u64 timestamp, |
---|
| 268 | + int napi_mode)); |
---|
267 | 269 | |
---|
268 | 270 | u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, |
---|
269 | 271 | struct mlx4_en_tx_ring *ring, |
---|
.. | .. |
---|
331 | 333 | |
---|
332 | 334 | return tx_info->nr_txbb; |
---|
333 | 335 | } |
---|
| 336 | + |
---|
| 337 | +INDIRECT_CALLABLE_DECLARE(u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv, |
---|
| 338 | + struct mlx4_en_tx_ring *ring, |
---|
| 339 | + int index, u64 timestamp, |
---|
| 340 | + int napi_mode)); |
---|
334 | 341 | |
---|
335 | 342 | u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv, |
---|
336 | 343 | struct mlx4_en_tx_ring *ring, |
---|
.. | .. |
---|
414 | 421 | queue_work(mdev->workqueue, &priv->restart_task); |
---|
415 | 422 | } |
---|
416 | 423 | |
---|
417 | | -bool mlx4_en_process_tx_cq(struct net_device *dev, |
---|
418 | | - struct mlx4_en_cq *cq, int napi_budget) |
---|
| 424 | +int mlx4_en_process_tx_cq(struct net_device *dev, |
---|
| 425 | + struct mlx4_en_cq *cq, int napi_budget) |
---|
419 | 426 | { |
---|
420 | 427 | struct mlx4_en_priv *priv = netdev_priv(dev); |
---|
421 | 428 | struct mlx4_cq *mcq = &cq->mcq; |
---|
.. | .. |
---|
437 | 444 | u32 ring_cons; |
---|
438 | 445 | |
---|
439 | 446 | if (unlikely(!priv->port_up)) |
---|
440 | | - return true; |
---|
| 447 | + return 0; |
---|
441 | 448 | |
---|
442 | 449 | netdev_txq_bql_complete_prefetchw(ring->tx_queue); |
---|
443 | 450 | |
---|
.. | .. |
---|
478 | 485 | timestamp = mlx4_en_get_cqe_ts(cqe); |
---|
479 | 486 | |
---|
480 | 487 | /* free next descriptor */ |
---|
481 | | - last_nr_txbb = ring->free_tx_desc( |
---|
| 488 | + last_nr_txbb = INDIRECT_CALL_2(ring->free_tx_desc, |
---|
| 489 | + mlx4_en_free_tx_desc, |
---|
| 490 | + mlx4_en_recycle_tx_desc, |
---|
482 | 491 | priv, ring, ring_index, |
---|
483 | 492 | timestamp, napi_budget); |
---|
484 | 493 | |
---|
.. | .. |
---|
509 | 518 | WRITE_ONCE(ring->cons, ring_cons + txbbs_skipped); |
---|
510 | 519 | |
---|
511 | 520 | if (cq->type == TX_XDP) |
---|
512 | | - return done < budget; |
---|
| 521 | + return done; |
---|
513 | 522 | |
---|
514 | 523 | netdev_tx_completed_queue(ring->tx_queue, packets, bytes); |
---|
515 | 524 | |
---|
.. | .. |
---|
521 | 530 | ring->wake_queue++; |
---|
522 | 531 | } |
---|
523 | 532 | |
---|
524 | | - return done < budget; |
---|
| 533 | + return done; |
---|
525 | 534 | } |
---|
526 | 535 | |
---|
527 | 536 | void mlx4_en_tx_irq(struct mlx4_cq *mcq) |
---|
.. | .. |
---|
541 | 550 | struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi); |
---|
542 | 551 | struct net_device *dev = cq->dev; |
---|
543 | 552 | struct mlx4_en_priv *priv = netdev_priv(dev); |
---|
544 | | - bool clean_complete; |
---|
| 553 | + int work_done; |
---|
545 | 554 | |
---|
546 | | - clean_complete = mlx4_en_process_tx_cq(dev, cq, budget); |
---|
547 | | - if (!clean_complete) |
---|
| 555 | + work_done = mlx4_en_process_tx_cq(dev, cq, budget); |
---|
| 556 | + if (work_done >= budget) |
---|
548 | 557 | return budget; |
---|
549 | 558 | |
---|
550 | | - napi_complete(napi); |
---|
551 | | - mlx4_en_arm_cq(priv, cq); |
---|
| 559 | + if (napi_complete_done(napi, work_done)) |
---|
| 560 | + mlx4_en_arm_cq(priv, cq); |
---|
552 | 561 | |
---|
553 | 562 | return 0; |
---|
554 | 563 | } |
---|
.. | .. |
---|
714 | 723 | } |
---|
715 | 724 | |
---|
716 | 725 | u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb, |
---|
717 | | - struct net_device *sb_dev, |
---|
718 | | - select_queue_fallback_t fallback) |
---|
| 726 | + struct net_device *sb_dev) |
---|
719 | 727 | { |
---|
720 | 728 | struct mlx4_en_priv *priv = netdev_priv(dev); |
---|
721 | 729 | u16 rings_p_up = priv->num_tx_rings_p_up; |
---|
722 | 730 | |
---|
723 | 731 | if (netdev_get_num_tc(dev)) |
---|
724 | | - return fallback(dev, skb, NULL); |
---|
| 732 | + return netdev_pick_tx(dev, skb, NULL); |
---|
725 | 733 | |
---|
726 | | - return fallback(dev, skb, NULL) % rings_p_up; |
---|
| 734 | + return netdev_pick_tx(dev, skb, NULL) % rings_p_up; |
---|
727 | 735 | } |
---|
728 | 736 | |
---|
729 | 737 | static void mlx4_bf_copy(void __iomem *dst, const void *src, |
---|
.. | .. |
---|
802 | 810 | |
---|
803 | 811 | /* Map fragments if any */ |
---|
804 | 812 | for (i_frag = shinfo->nr_frags - 1; i_frag >= 0; i_frag--) { |
---|
805 | | - const struct skb_frag_struct *frag; |
---|
806 | | - |
---|
807 | | - frag = &shinfo->frags[i_frag]; |
---|
| 813 | + const skb_frag_t *frag = &shinfo->frags[i_frag]; |
---|
808 | 814 | byte_count = skb_frag_size(frag); |
---|
809 | 815 | dma = skb_frag_dma_map(ddev, frag, |
---|
810 | 816 | 0, byte_count, |
---|
.. | .. |
---|
1032 | 1038 | ring->packets++; |
---|
1033 | 1039 | } |
---|
1034 | 1040 | ring->bytes += tx_info->nr_bytes; |
---|
1035 | | - netdev_tx_sent_queue(ring->tx_queue, tx_info->nr_bytes); |
---|
1036 | 1041 | AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, skb->len); |
---|
1037 | 1042 | |
---|
1038 | 1043 | if (tx_info->inl) |
---|
.. | .. |
---|
1070 | 1075 | netif_tx_stop_queue(ring->tx_queue); |
---|
1071 | 1076 | ring->queue_stopped++; |
---|
1072 | 1077 | } |
---|
1073 | | - send_doorbell = !skb->xmit_more || netif_xmit_stopped(ring->tx_queue); |
---|
| 1078 | + |
---|
| 1079 | + send_doorbell = __netdev_tx_sent_queue(ring->tx_queue, |
---|
| 1080 | + tx_info->nr_bytes, |
---|
| 1081 | + netdev_xmit_more()); |
---|
1074 | 1082 | |
---|
1075 | 1083 | real_size = (real_size / 16) & 0x3f; |
---|
1076 | 1084 | |
---|
.. | .. |
---|
1093 | 1101 | */ |
---|
1094 | 1102 | smp_rmb(); |
---|
1095 | 1103 | |
---|
1096 | | - ring_cons = READ_ONCE(ring->cons); |
---|
1097 | 1104 | if (unlikely(!mlx4_en_is_tx_ring_full(ring))) { |
---|
1098 | 1105 | netif_tx_wake_queue(ring->tx_queue); |
---|
1099 | 1106 | ring->wake_queue++; |
---|