| .. | .. |
|---|
| 43 | 43 | #include <linux/ip.h> |
|---|
| 44 | 44 | #include <linux/ipv6.h> |
|---|
| 45 | 45 | #include <linux/moduleparam.h> |
|---|
| 46 | +#include <linux/indirect_call_wrapper.h> |
|---|
| 46 | 47 | |
|---|
| 47 | 48 | #include "mlx4_en.h" |
|---|
| 48 | 49 | |
|---|
| .. | .. |
|---|
| 57 | 58 | |
|---|
| 58 | 59 | ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node); |
|---|
| 59 | 60 | if (!ring) { |
|---|
| 60 | | - ring = kzalloc(sizeof(*ring), GFP_KERNEL); |
|---|
| 61 | | - if (!ring) { |
|---|
| 62 | | - en_err(priv, "Failed allocating TX ring\n"); |
|---|
| 63 | | - return -ENOMEM; |
|---|
| 64 | | - } |
|---|
| 61 | + en_err(priv, "Failed allocating TX ring\n"); |
|---|
| 62 | + return -ENOMEM; |
|---|
| 65 | 63 | } |
|---|
| 66 | 64 | |
|---|
| 67 | 65 | ring->size = size; |
|---|
| .. | .. |
|---|
| 264 | 262 | } |
|---|
| 265 | 263 | } |
|---|
| 266 | 264 | |
|---|
| 265 | +INDIRECT_CALLABLE_DECLARE(u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, |
|---|
| 266 | + struct mlx4_en_tx_ring *ring, |
|---|
| 267 | + int index, u64 timestamp, |
|---|
| 268 | + int napi_mode)); |
|---|
| 267 | 269 | |
|---|
| 268 | 270 | u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, |
|---|
| 269 | 271 | struct mlx4_en_tx_ring *ring, |
|---|
| .. | .. |
|---|
| 331 | 333 | |
|---|
| 332 | 334 | return tx_info->nr_txbb; |
|---|
| 333 | 335 | } |
|---|
| 336 | + |
|---|
| 337 | +INDIRECT_CALLABLE_DECLARE(u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv, |
|---|
| 338 | + struct mlx4_en_tx_ring *ring, |
|---|
| 339 | + int index, u64 timestamp, |
|---|
| 340 | + int napi_mode)); |
|---|
| 334 | 341 | |
|---|
| 335 | 342 | u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv, |
|---|
| 336 | 343 | struct mlx4_en_tx_ring *ring, |
|---|
| .. | .. |
|---|
| 414 | 421 | queue_work(mdev->workqueue, &priv->restart_task); |
|---|
| 415 | 422 | } |
|---|
| 416 | 423 | |
|---|
| 417 | | -bool mlx4_en_process_tx_cq(struct net_device *dev, |
|---|
| 418 | | - struct mlx4_en_cq *cq, int napi_budget) |
|---|
| 424 | +int mlx4_en_process_tx_cq(struct net_device *dev, |
|---|
| 425 | + struct mlx4_en_cq *cq, int napi_budget) |
|---|
| 419 | 426 | { |
|---|
| 420 | 427 | struct mlx4_en_priv *priv = netdev_priv(dev); |
|---|
| 421 | 428 | struct mlx4_cq *mcq = &cq->mcq; |
|---|
| .. | .. |
|---|
| 437 | 444 | u32 ring_cons; |
|---|
| 438 | 445 | |
|---|
| 439 | 446 | if (unlikely(!priv->port_up)) |
|---|
| 440 | | - return true; |
|---|
| 447 | + return 0; |
|---|
| 441 | 448 | |
|---|
| 442 | 449 | netdev_txq_bql_complete_prefetchw(ring->tx_queue); |
|---|
| 443 | 450 | |
|---|
| .. | .. |
|---|
| 478 | 485 | timestamp = mlx4_en_get_cqe_ts(cqe); |
|---|
| 479 | 486 | |
|---|
| 480 | 487 | /* free next descriptor */ |
|---|
| 481 | | - last_nr_txbb = ring->free_tx_desc( |
|---|
| 488 | + last_nr_txbb = INDIRECT_CALL_2(ring->free_tx_desc, |
|---|
| 489 | + mlx4_en_free_tx_desc, |
|---|
| 490 | + mlx4_en_recycle_tx_desc, |
|---|
| 482 | 491 | priv, ring, ring_index, |
|---|
| 483 | 492 | timestamp, napi_budget); |
|---|
| 484 | 493 | |
|---|
| .. | .. |
|---|
| 509 | 518 | WRITE_ONCE(ring->cons, ring_cons + txbbs_skipped); |
|---|
| 510 | 519 | |
|---|
| 511 | 520 | if (cq->type == TX_XDP) |
|---|
| 512 | | - return done < budget; |
|---|
| 521 | + return done; |
|---|
| 513 | 522 | |
|---|
| 514 | 523 | netdev_tx_completed_queue(ring->tx_queue, packets, bytes); |
|---|
| 515 | 524 | |
|---|
| .. | .. |
|---|
| 521 | 530 | ring->wake_queue++; |
|---|
| 522 | 531 | } |
|---|
| 523 | 532 | |
|---|
| 524 | | - return done < budget; |
|---|
| 533 | + return done; |
|---|
| 525 | 534 | } |
|---|
| 526 | 535 | |
|---|
| 527 | 536 | void mlx4_en_tx_irq(struct mlx4_cq *mcq) |
|---|
| .. | .. |
|---|
| 541 | 550 | struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi); |
|---|
| 542 | 551 | struct net_device *dev = cq->dev; |
|---|
| 543 | 552 | struct mlx4_en_priv *priv = netdev_priv(dev); |
|---|
| 544 | | - bool clean_complete; |
|---|
| 553 | + int work_done; |
|---|
| 545 | 554 | |
|---|
| 546 | | - clean_complete = mlx4_en_process_tx_cq(dev, cq, budget); |
|---|
| 547 | | - if (!clean_complete) |
|---|
| 555 | + work_done = mlx4_en_process_tx_cq(dev, cq, budget); |
|---|
| 556 | + if (work_done >= budget) |
|---|
| 548 | 557 | return budget; |
|---|
| 549 | 558 | |
|---|
| 550 | | - napi_complete(napi); |
|---|
| 551 | | - mlx4_en_arm_cq(priv, cq); |
|---|
| 559 | + if (napi_complete_done(napi, work_done)) |
|---|
| 560 | + mlx4_en_arm_cq(priv, cq); |
|---|
| 552 | 561 | |
|---|
| 553 | 562 | return 0; |
|---|
| 554 | 563 | } |
|---|
| .. | .. |
|---|
| 714 | 723 | } |
|---|
| 715 | 724 | |
|---|
| 716 | 725 | u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb, |
|---|
| 717 | | - struct net_device *sb_dev, |
|---|
| 718 | | - select_queue_fallback_t fallback) |
|---|
| 726 | + struct net_device *sb_dev) |
|---|
| 719 | 727 | { |
|---|
| 720 | 728 | struct mlx4_en_priv *priv = netdev_priv(dev); |
|---|
| 721 | 729 | u16 rings_p_up = priv->num_tx_rings_p_up; |
|---|
| 722 | 730 | |
|---|
| 723 | 731 | if (netdev_get_num_tc(dev)) |
|---|
| 724 | | - return fallback(dev, skb, NULL); |
|---|
| 732 | + return netdev_pick_tx(dev, skb, NULL); |
|---|
| 725 | 733 | |
|---|
| 726 | | - return fallback(dev, skb, NULL) % rings_p_up; |
|---|
| 734 | + return netdev_pick_tx(dev, skb, NULL) % rings_p_up; |
|---|
| 727 | 735 | } |
|---|
| 728 | 736 | |
|---|
| 729 | 737 | static void mlx4_bf_copy(void __iomem *dst, const void *src, |
|---|
| .. | .. |
|---|
| 802 | 810 | |
|---|
| 803 | 811 | /* Map fragments if any */ |
|---|
| 804 | 812 | for (i_frag = shinfo->nr_frags - 1; i_frag >= 0; i_frag--) { |
|---|
| 805 | | - const struct skb_frag_struct *frag; |
|---|
| 806 | | - |
|---|
| 807 | | - frag = &shinfo->frags[i_frag]; |
|---|
| 813 | + const skb_frag_t *frag = &shinfo->frags[i_frag]; |
|---|
| 808 | 814 | byte_count = skb_frag_size(frag); |
|---|
| 809 | 815 | dma = skb_frag_dma_map(ddev, frag, |
|---|
| 810 | 816 | 0, byte_count, |
|---|
| .. | .. |
|---|
| 1032 | 1038 | ring->packets++; |
|---|
| 1033 | 1039 | } |
|---|
| 1034 | 1040 | ring->bytes += tx_info->nr_bytes; |
|---|
| 1035 | | - netdev_tx_sent_queue(ring->tx_queue, tx_info->nr_bytes); |
|---|
| 1036 | 1041 | AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, skb->len); |
|---|
| 1037 | 1042 | |
|---|
| 1038 | 1043 | if (tx_info->inl) |
|---|
| .. | .. |
|---|
| 1070 | 1075 | netif_tx_stop_queue(ring->tx_queue); |
|---|
| 1071 | 1076 | ring->queue_stopped++; |
|---|
| 1072 | 1077 | } |
|---|
| 1073 | | - send_doorbell = !skb->xmit_more || netif_xmit_stopped(ring->tx_queue); |
|---|
| 1078 | + |
|---|
| 1079 | + send_doorbell = __netdev_tx_sent_queue(ring->tx_queue, |
|---|
| 1080 | + tx_info->nr_bytes, |
|---|
| 1081 | + netdev_xmit_more()); |
|---|
| 1074 | 1082 | |
|---|
| 1075 | 1083 | real_size = (real_size / 16) & 0x3f; |
|---|
| 1076 | 1084 | |
|---|
| .. | .. |
|---|
| 1093 | 1101 | */ |
|---|
| 1094 | 1102 | smp_rmb(); |
|---|
| 1095 | 1103 | |
|---|
| 1096 | | - ring_cons = READ_ONCE(ring->cons); |
|---|
| 1097 | 1104 | if (unlikely(!mlx4_en_is_tx_ring_full(ring))) { |
|---|
| 1098 | 1105 | netif_tx_wake_queue(ring->tx_queue); |
|---|
| 1099 | 1106 | ring->wake_queue++; |
|---|