mk-rootfs.sh

hc

2023-12-11 072de836f53be56a70cecf70b43ae43b7ce17376

 kernel/kernel/bpf/percpu_freelist.c
..
..
@@ -1,8 +1,5 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
1
2
 /* Copyright (c) 2016 Facebook
2
- *
3
- * This program is free software; you can redistribute it and/or
4
- * modify it under the terms of version 2 of the GNU General Public
5
- * License as published by the Free Software Foundation.
6
3
  */
7
4
 #include "percpu_freelist.h"
8
5
 
..
..
@@ -20,6 +17,8 @@
20
17
 		raw_spin_lock_init(&head->lock);
21
18
 		head->first = NULL;
22
19
 	}
20
+	raw_spin_lock_init(&s->extralist.lock);
21
+	s->extralist.first = NULL;
23
22
 	return 0;
24
23
 }
25
24
 
..
..
@@ -28,21 +27,65 @@
28
27
 	free_percpu(s->freelist);
29
28
 }
30
29
 
30
+static inline void pcpu_freelist_push_node(struct pcpu_freelist_head *head,
31
+					   struct pcpu_freelist_node *node)
32
+{
33
+	node->next = head->first;
34
+	head->first = node;
35
+}
36
+
31
37
 static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head,
32
38
 					 struct pcpu_freelist_node *node)
33
39
 {
34
40
 	raw_spin_lock(&head->lock);
35
-	node->next = head->first;
36
-	head->first = node;
41
+	pcpu_freelist_push_node(head, node);
37
42
 	raw_spin_unlock(&head->lock);
43
+}
44
+
45
+static inline bool pcpu_freelist_try_push_extra(struct pcpu_freelist *s,
46
+						struct pcpu_freelist_node *node)
47
+{
48
+	if (!raw_spin_trylock(&s->extralist.lock))
49
+		return false;
50
+
51
+	pcpu_freelist_push_node(&s->extralist, node);
52
+	raw_spin_unlock(&s->extralist.lock);
53
+	return true;
54
+}
55
+
56
+static inline void ___pcpu_freelist_push_nmi(struct pcpu_freelist *s,
57
+					     struct pcpu_freelist_node *node)
58
+{
59
+	int cpu, orig_cpu;
60
+
61
+	orig_cpu = cpu = raw_smp_processor_id();
62
+	while (1) {
63
+		struct pcpu_freelist_head *head;
64
+
65
+		head = per_cpu_ptr(s->freelist, cpu);
66
+		if (raw_spin_trylock(&head->lock)) {
67
+			pcpu_freelist_push_node(head, node);
68
+			raw_spin_unlock(&head->lock);
69
+			return;
70
+		}
71
+		cpu = cpumask_next(cpu, cpu_possible_mask);
72
+		if (cpu >= nr_cpu_ids)
73
+			cpu = 0;
74
+
75
+		/* cannot lock any per cpu lock, try extralist */
76
+		if (cpu == orig_cpu &&
77
+		    pcpu_freelist_try_push_extra(s, node))
78
+			return;
79
+	}
38
80
 }
39
81
 
40
82
 void __pcpu_freelist_push(struct pcpu_freelist *s,
41
83
 			struct pcpu_freelist_node *node)
42
84
 {
43
-	struct pcpu_freelist_head *head = this_cpu_ptr(s->freelist);
44
-
45
-	___pcpu_freelist_push(head, node);
85
+	if (in_nmi())
86
+		___pcpu_freelist_push_nmi(s, node);
87
+	else
88
+		___pcpu_freelist_push(this_cpu_ptr(s->freelist), node);
46
89
 }
47
90
 
48
91
 void pcpu_freelist_push(struct pcpu_freelist *s,
..
..
@@ -59,32 +102,25 @@
59
102
 			    u32 nr_elems)
60
103
 {
61
104
 	struct pcpu_freelist_head *head;
62
-	unsigned long flags;
63
-	int i, cpu, pcpu_entries;
105
+	unsigned int cpu, cpu_idx, i, j, n, m;
64
106
 
65
-	pcpu_entries = nr_elems / num_possible_cpus() + 1;
66
-	i = 0;
107
+	n = nr_elems / num_possible_cpus();
108
+	m = nr_elems % num_possible_cpus();
67
109
 
68
-	/* disable irq to workaround lockdep false positive
69
-	 * in bpf usage pcpu_freelist_populate() will never race
70
-	 * with pcpu_freelist_push()
71
-	 */
72
-	local_irq_save(flags);
110
+	cpu_idx = 0;
73
111
 	for_each_possible_cpu(cpu) {
74
-again:
75
112
 		head = per_cpu_ptr(s->freelist, cpu);
76
-		___pcpu_freelist_push(head, buf);
77
-		i++;
78
-		buf += elem_size;
79
-		if (i == nr_elems)
80
-			break;
81
-		if (i % pcpu_entries)
82
-			goto again;
113
+		j = n + (cpu_idx < m ? 1 : 0);
114
+		for (i = 0; i < j; i++) {
115
+			/* No locking required as this is not visible yet. */
116
+			pcpu_freelist_push_node(head, buf);
117
+			buf += elem_size;
118
+		}
119
+		cpu_idx++;
83
120
 	}
84
-	local_irq_restore(flags);
85
121
 }
86
122
 
87
-struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *s)
123
+static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s)
88
124
 {
89
125
 	struct pcpu_freelist_head *head;
90
126
 	struct pcpu_freelist_node *node;
..
..
@@ -105,8 +141,59 @@
105
141
 		if (cpu >= nr_cpu_ids)
106
142
 			cpu = 0;
107
143
 		if (cpu == orig_cpu)
108
-			return NULL;
144
+			break;
109
145
 	}
146
+
147
+	/* per cpu lists are all empty, try extralist */
148
+	raw_spin_lock(&s->extralist.lock);
149
+	node = s->extralist.first;
150
+	if (node)
151
+		s->extralist.first = node->next;
152
+	raw_spin_unlock(&s->extralist.lock);
153
+	return node;
154
+}
155
+
156
+static struct pcpu_freelist_node *
157
+___pcpu_freelist_pop_nmi(struct pcpu_freelist *s)
158
+{
159
+	struct pcpu_freelist_head *head;
160
+	struct pcpu_freelist_node *node;
161
+	int orig_cpu, cpu;
162
+
163
+	orig_cpu = cpu = raw_smp_processor_id();
164
+	while (1) {
165
+		head = per_cpu_ptr(s->freelist, cpu);
166
+		if (raw_spin_trylock(&head->lock)) {
167
+			node = head->first;
168
+			if (node) {
169
+				head->first = node->next;
170
+				raw_spin_unlock(&head->lock);
171
+				return node;
172
+			}
173
+			raw_spin_unlock(&head->lock);
174
+		}
175
+		cpu = cpumask_next(cpu, cpu_possible_mask);
176
+		if (cpu >= nr_cpu_ids)
177
+			cpu = 0;
178
+		if (cpu == orig_cpu)
179
+			break;
180
+	}
181
+
182
+	/* cannot pop from per cpu lists, try extralist */
183
+	if (!raw_spin_trylock(&s->extralist.lock))
184
+		return NULL;
185
+	node = s->extralist.first;
186
+	if (node)
187
+		s->extralist.first = node->next;
188
+	raw_spin_unlock(&s->extralist.lock);
189
+	return node;
190
+}
191
+
192
+struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *s)
193
+{
194
+	if (in_nmi())
195
+		return ___pcpu_freelist_pop_nmi(s);
196
+	return ___pcpu_freelist_pop(s);
110
197
 }
111
198
 
112
199
 struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)