hc
2024-01-05 071106ecf68c401173c58808b1cf5f68cc50d390
kernel/drivers/net/wireguard/receive.c
....@@ -116,8 +116,8 @@
116116 return;
117117 }
118118
119
- under_load = skb_queue_len(&wg->incoming_handshakes) >=
120
- MAX_QUEUED_INCOMING_HANDSHAKES / 8;
119
+ under_load = atomic_read(&wg->handshake_queue_len) >=
120
+ MAX_QUEUED_INCOMING_HANDSHAKES / 8;
121121 if (under_load) {
122122 last_under_load = ktime_get_coarse_boottime_ns();
123123 } else if (last_under_load) {
....@@ -212,13 +212,14 @@
212212
213213 void wg_packet_handshake_receive_worker(struct work_struct *work)
214214 {
215
- struct wg_device *wg = container_of(work, struct multicore_worker,
216
- work)->ptr;
215
+ struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->ptr;
216
+ struct wg_device *wg = container_of(queue, struct wg_device, handshake_queue);
217217 struct sk_buff *skb;
218218
219
- while ((skb = skb_dequeue(&wg->incoming_handshakes)) != NULL) {
219
+ while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) {
220220 wg_receive_handshake_packet(wg, skb);
221221 dev_kfree_skb(skb);
222
+ atomic_dec(&wg->handshake_queue_len);
222223 cond_resched();
223224 }
224225 }
....@@ -444,7 +445,6 @@
444445 int wg_packet_rx_poll(struct napi_struct *napi, int budget)
445446 {
446447 struct wg_peer *peer = container_of(napi, struct wg_peer, napi);
447
- struct crypt_queue *queue = &peer->rx_queue;
448448 struct noise_keypair *keypair;
449449 struct endpoint endpoint;
450450 enum packet_state state;
....@@ -455,11 +455,10 @@
455455 if (unlikely(budget <= 0))
456456 return 0;
457457
458
- while ((skb = __ptr_ring_peek(&queue->ring)) != NULL &&
458
+ while ((skb = wg_prev_queue_peek(&peer->rx_queue)) != NULL &&
459459 (state = atomic_read_acquire(&PACKET_CB(skb)->state)) !=
460460 PACKET_STATE_UNCRYPTED) {
461
- __ptr_ring_discard_one(&queue->ring);
462
- peer = PACKET_PEER(skb);
461
+ wg_prev_queue_drop_peeked(&peer->rx_queue);
463462 keypair = PACKET_CB(skb)->keypair;
464463 free = true;
465464
....@@ -508,7 +507,7 @@
508507 enum packet_state state =
509508 likely(decrypt_packet(skb, PACKET_CB(skb)->keypair)) ?
510509 PACKET_STATE_CRYPTED : PACKET_STATE_DEAD;
511
- wg_queue_enqueue_per_peer_napi(skb, state);
510
+ wg_queue_enqueue_per_peer_rx(skb, state);
512511 if (need_resched())
513512 cond_resched();
514513 }
....@@ -531,12 +530,10 @@
531530 if (unlikely(READ_ONCE(peer->is_dead)))
532531 goto err;
533532
534
- ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue,
535
- &peer->rx_queue, skb,
536
- wg->packet_crypt_wq,
537
- &wg->decrypt_queue.last_cpu);
533
+ ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, &peer->rx_queue, skb,
534
+ wg->packet_crypt_wq);
538535 if (unlikely(ret == -EPIPE))
539
- wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD);
536
+ wg_queue_enqueue_per_peer_rx(skb, PACKET_STATE_DEAD);
540537 if (likely(!ret || ret == -EPIPE)) {
541538 rcu_read_unlock_bh();
542539 return;
....@@ -557,22 +554,28 @@
557554 case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION):
558555 case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE):
559556 case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): {
560
- int cpu;
557
+ int cpu, ret = -EBUSY;
561558
562
- if (skb_queue_len(&wg->incoming_handshakes) >
563
- MAX_QUEUED_INCOMING_HANDSHAKES ||
564
- unlikely(!rng_is_initialized())) {
559
+ if (unlikely(!rng_is_initialized()))
560
+ goto drop;
561
+ if (atomic_read(&wg->handshake_queue_len) > MAX_QUEUED_INCOMING_HANDSHAKES / 2) {
562
+ if (spin_trylock_bh(&wg->handshake_queue.ring.producer_lock)) {
563
+ ret = __ptr_ring_produce(&wg->handshake_queue.ring, skb);
564
+ spin_unlock_bh(&wg->handshake_queue.ring.producer_lock);
565
+ }
566
+ } else
567
+ ret = ptr_ring_produce_bh(&wg->handshake_queue.ring, skb);
568
+ if (ret) {
569
+ drop:
565570 net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n",
566571 wg->dev->name, skb);
567572 goto err;
568573 }
569
- skb_queue_tail(&wg->incoming_handshakes, skb);
570
- /* Queues up a call to packet_process_queued_handshake_
571
- * packets(skb):
572
- */
573
- cpu = wg_cpumask_next_online(&wg->incoming_handshake_cpu);
574
+ atomic_inc(&wg->handshake_queue_len);
575
+ cpu = wg_cpumask_next_online(&wg->handshake_queue.last_cpu);
576
+ /* Queues up a call to packet_process_queued_handshake_packets(skb): */
574577 queue_work_on(cpu, wg->handshake_receive_wq,
575
- &per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work);
578
+ &per_cpu_ptr(wg->handshake_queue.worker, cpu)->work);
576579 break;
577580 }
578581 case cpu_to_le32(MESSAGE_DATA):