.. | .. |
---|
222 | 222 | static inline void rps_lock(struct softnet_data *sd) |
---|
223 | 223 | { |
---|
224 | 224 | #ifdef CONFIG_RPS |
---|
225 | | - raw_spin_lock(&sd->input_pkt_queue.raw_lock); |
---|
| 225 | + spin_lock(&sd->input_pkt_queue.lock); |
---|
226 | 226 | #endif |
---|
227 | 227 | } |
---|
228 | 228 | |
---|
229 | 229 | static inline void rps_unlock(struct softnet_data *sd) |
---|
230 | 230 | { |
---|
231 | 231 | #ifdef CONFIG_RPS |
---|
232 | | - raw_spin_unlock(&sd->input_pkt_queue.raw_lock); |
---|
| 232 | + spin_unlock(&sd->input_pkt_queue.lock); |
---|
233 | 233 | #endif |
---|
234 | 234 | } |
---|
235 | 235 | |
---|
.. | .. |
---|
2635 | 2635 | bool active = false; |
---|
2636 | 2636 | unsigned int nr_ids; |
---|
2637 | 2637 | |
---|
| 2638 | + WARN_ON_ONCE(index >= dev->num_tx_queues); |
---|
| 2639 | + |
---|
2638 | 2640 | if (dev->num_tc) { |
---|
2639 | 2641 | /* Do not allow XPS on subordinate device directly */ |
---|
2640 | 2642 | num_tc = dev->num_tc; |
---|
.. | .. |
---|
3055 | 3057 | sd->output_queue_tailp = &q->next_sched; |
---|
3056 | 3058 | raise_softirq_irqoff(NET_TX_SOFTIRQ); |
---|
3057 | 3059 | local_irq_restore(flags); |
---|
3058 | | - preempt_check_resched_rt(); |
---|
3059 | 3060 | } |
---|
3060 | 3061 | |
---|
3061 | 3062 | void __netif_schedule(struct Qdisc *q) |
---|
.. | .. |
---|
3118 | 3119 | __this_cpu_write(softnet_data.completion_queue, skb); |
---|
3119 | 3120 | raise_softirq_irqoff(NET_TX_SOFTIRQ); |
---|
3120 | 3121 | local_irq_restore(flags); |
---|
3121 | | - preempt_check_resched_rt(); |
---|
3122 | 3122 | } |
---|
3123 | 3123 | EXPORT_SYMBOL(__dev_kfree_skb_irq); |
---|
3124 | 3124 | |
---|
.. | .. |
---|
3126 | 3126 | { |
---|
3127 | 3127 | if (in_irq() || irqs_disabled()) |
---|
3128 | 3128 | __dev_kfree_skb_irq(skb, reason); |
---|
| 3129 | + else if (unlikely(reason == SKB_REASON_DROPPED)) |
---|
| 3130 | + kfree_skb(skb); |
---|
3129 | 3131 | else |
---|
3130 | | - dev_kfree_skb(skb); |
---|
| 3132 | + consume_skb(skb); |
---|
3131 | 3133 | } |
---|
3132 | 3134 | EXPORT_SYMBOL(__dev_kfree_skb_any); |
---|
3133 | 3135 | |
---|
.. | .. |
---|
3325 | 3327 | type = eth->h_proto; |
---|
3326 | 3328 | } |
---|
3327 | 3329 | |
---|
3328 | | - return __vlan_get_protocol(skb, type, depth); |
---|
| 3330 | + return vlan_get_protocol_and_depth(skb, type, depth); |
---|
3329 | 3331 | } |
---|
3330 | 3332 | |
---|
3331 | 3333 | /** |
---|
.. | .. |
---|
3638 | 3640 | int skb_csum_hwoffload_help(struct sk_buff *skb, |
---|
3639 | 3641 | const netdev_features_t features) |
---|
3640 | 3642 | { |
---|
3641 | | - if (unlikely(skb->csum_not_inet)) |
---|
| 3643 | + if (unlikely(skb_csum_is_sctp(skb))) |
---|
3642 | 3644 | return !!(features & NETIF_F_SCTP_CRC) ? 0 : |
---|
3643 | 3645 | skb_crc32c_csum_help(skb); |
---|
3644 | 3646 | |
---|
.. | .. |
---|
3797 | 3799 | * This permits qdisc->running owner to get the lock more |
---|
3798 | 3800 | * often and dequeue packets faster. |
---|
3799 | 3801 | */ |
---|
3800 | | -#ifdef CONFIG_PREEMPT_RT |
---|
3801 | | - contended = true; |
---|
3802 | | -#else |
---|
3803 | 3802 | contended = qdisc_is_running(q); |
---|
3804 | | -#endif |
---|
3805 | 3803 | if (unlikely(contended)) |
---|
3806 | 3804 | spin_lock(&q->busylock); |
---|
3807 | 3805 | |
---|
.. | .. |
---|
4392 | 4390 | u32 next_cpu; |
---|
4393 | 4391 | u32 ident; |
---|
4394 | 4392 | |
---|
4395 | | - /* First check into global flow table if there is a match */ |
---|
4396 | | - ident = sock_flow_table->ents[hash & sock_flow_table->mask]; |
---|
| 4393 | + /* First check into global flow table if there is a match. |
---|
| 4394 | + * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow(). |
---|
| 4395 | + */ |
---|
| 4396 | + ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]); |
---|
4397 | 4397 | if ((ident ^ hash) & ~rps_cpu_mask) |
---|
4398 | 4398 | goto try_rps; |
---|
4399 | 4399 | |
---|
.. | .. |
---|
4601 | 4601 | rps_unlock(sd); |
---|
4602 | 4602 | |
---|
4603 | 4603 | local_irq_restore(flags); |
---|
4604 | | - preempt_check_resched_rt(); |
---|
4605 | 4604 | |
---|
4606 | 4605 | atomic_long_inc(&skb->dev->rx_dropped); |
---|
4607 | 4606 | kfree_skb(skb); |
---|
.. | .. |
---|
4817 | 4816 | struct rps_dev_flow voidflow, *rflow = &voidflow; |
---|
4818 | 4817 | int cpu; |
---|
4819 | 4818 | |
---|
4820 | | - migrate_disable(); |
---|
| 4819 | + preempt_disable(); |
---|
4821 | 4820 | rcu_read_lock(); |
---|
4822 | 4821 | |
---|
4823 | 4822 | cpu = get_rps_cpu(skb->dev, skb, &rflow); |
---|
.. | .. |
---|
4827 | 4826 | ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); |
---|
4828 | 4827 | |
---|
4829 | 4828 | rcu_read_unlock(); |
---|
4830 | | - migrate_enable(); |
---|
| 4829 | + preempt_enable(); |
---|
4831 | 4830 | } else |
---|
4832 | 4831 | #endif |
---|
4833 | 4832 | { |
---|
4834 | 4833 | unsigned int qtail; |
---|
4835 | 4834 | |
---|
4836 | | - ret = enqueue_to_backlog(skb, get_cpu_light(), &qtail); |
---|
4837 | | - put_cpu_light(); |
---|
| 4835 | + ret = enqueue_to_backlog(skb, get_cpu(), &qtail); |
---|
| 4836 | + put_cpu(); |
---|
4838 | 4837 | } |
---|
4839 | 4838 | return ret; |
---|
4840 | 4839 | } |
---|
.. | .. |
---|
4873 | 4872 | |
---|
4874 | 4873 | trace_netif_rx_ni_entry(skb); |
---|
4875 | 4874 | |
---|
4876 | | - local_bh_disable(); |
---|
| 4875 | + preempt_disable(); |
---|
4877 | 4876 | err = netif_rx_internal(skb); |
---|
4878 | | - local_bh_enable(); |
---|
| 4877 | + if (local_softirq_pending()) |
---|
| 4878 | + do_softirq(); |
---|
| 4879 | + preempt_enable(); |
---|
4879 | 4880 | trace_netif_rx_ni_exit(err); |
---|
4880 | 4881 | |
---|
4881 | 4882 | return err; |
---|
.. | .. |
---|
6119 | 6120 | |
---|
6120 | 6121 | static void napi_skb_free_stolen_head(struct sk_buff *skb) |
---|
6121 | 6122 | { |
---|
| 6123 | + nf_reset_ct(skb); |
---|
6122 | 6124 | skb_dst_drop(skb); |
---|
6123 | 6125 | skb_ext_put(skb); |
---|
6124 | 6126 | kmem_cache_free(skbuff_head_cache, skb); |
---|
.. | .. |
---|
6351 | 6353 | sd->rps_ipi_list = NULL; |
---|
6352 | 6354 | |
---|
6353 | 6355 | local_irq_enable(); |
---|
6354 | | - preempt_check_resched_rt(); |
---|
6355 | 6356 | |
---|
6356 | 6357 | /* Send pending IPI's to kick RPS processing on remote cpus. */ |
---|
6357 | 6358 | net_rps_send_ipi(remsd); |
---|
6358 | 6359 | } else |
---|
6359 | 6360 | #endif |
---|
6360 | 6361 | local_irq_enable(); |
---|
6361 | | - preempt_check_resched_rt(); |
---|
6362 | 6362 | } |
---|
6363 | 6363 | |
---|
6364 | 6364 | static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) |
---|
.. | .. |
---|
6436 | 6436 | local_irq_save(flags); |
---|
6437 | 6437 | ____napi_schedule(this_cpu_ptr(&softnet_data), n); |
---|
6438 | 6438 | local_irq_restore(flags); |
---|
6439 | | - preempt_check_resched_rt(); |
---|
6440 | 6439 | } |
---|
6441 | 6440 | EXPORT_SYMBOL(__napi_schedule); |
---|
6442 | 6441 | |
---|
.. | .. |
---|
10306 | 10305 | BUG_ON(!list_empty(&dev->ptype_specific)); |
---|
10307 | 10306 | WARN_ON(rcu_access_pointer(dev->ip_ptr)); |
---|
10308 | 10307 | WARN_ON(rcu_access_pointer(dev->ip6_ptr)); |
---|
10309 | | -#if IS_ENABLED(CONFIG_DECNET) |
---|
10310 | | - WARN_ON(dev->dn_ptr); |
---|
10311 | | -#endif |
---|
| 10308 | + |
---|
10312 | 10309 | if (dev->priv_destructor) |
---|
10313 | 10310 | dev->priv_destructor(dev); |
---|
10314 | 10311 | if (dev->needs_free_netdev) |
---|
.. | .. |
---|
10978 | 10975 | |
---|
10979 | 10976 | raise_softirq_irqoff(NET_TX_SOFTIRQ); |
---|
10980 | 10977 | local_irq_enable(); |
---|
10981 | | - preempt_check_resched_rt(); |
---|
10982 | 10978 | |
---|
10983 | 10979 | #ifdef CONFIG_RPS |
---|
10984 | 10980 | remsd = oldsd->rps_ipi_list; |
---|
.. | .. |
---|
10992 | 10988 | netif_rx_ni(skb); |
---|
10993 | 10989 | input_queue_head_incr(oldsd); |
---|
10994 | 10990 | } |
---|
10995 | | - while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) { |
---|
| 10991 | + while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { |
---|
10996 | 10992 | netif_rx_ni(skb); |
---|
10997 | 10993 | input_queue_head_incr(oldsd); |
---|
10998 | 10994 | } |
---|
.. | .. |
---|
11308 | 11304 | |
---|
11309 | 11305 | INIT_WORK(flush, flush_backlog); |
---|
11310 | 11306 | |
---|
11311 | | - skb_queue_head_init_raw(&sd->input_pkt_queue); |
---|
| 11307 | + skb_queue_head_init(&sd->input_pkt_queue); |
---|
11312 | 11308 | skb_queue_head_init(&sd->process_queue); |
---|
11313 | 11309 | #ifdef CONFIG_XFRM_OFFLOAD |
---|
11314 | 11310 | skb_queue_head_init(&sd->xfrm_backlog); |
---|