add new system file

hc

2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c

 kernel/fs/erofs/zdata.c
..
..
@@ -7,7 +7,7 @@
7
7
 #include "zdata.h"
8
8
 #include "compress.h"
9
9
 #include <linux/prefetch.h>
10
-
10
+#include <linux/cpuhotplug.h>
11
11
 #include <trace/events/erofs.h>
12
12
 
13
13
 /*
..
..
@@ -125,24 +125,128 @@
125
125
 
126
126
 static struct workqueue_struct *z_erofs_workqueue __read_mostly;
127
127
 
128
-void z_erofs_exit_zip_subsystem(void)
128
+#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD
129
+static struct kthread_worker __rcu **z_erofs_pcpu_workers;
130
+
131
+static void erofs_destroy_percpu_workers(void)
129
132
 {
130
-	destroy_workqueue(z_erofs_workqueue);
131
-	z_erofs_destroy_pcluster_pool();
133
+	struct kthread_worker *worker;
134
+	unsigned int cpu;
135
+
136
+	for_each_possible_cpu(cpu) {
137
+		worker = rcu_dereference_protected(
138
+					z_erofs_pcpu_workers[cpu], 1);
139
+		rcu_assign_pointer(z_erofs_pcpu_workers[cpu], NULL);
140
+		if (worker)
141
+			kthread_destroy_worker(worker);
142
+	}
143
+	kfree(z_erofs_pcpu_workers);
132
144
 }
133
145
 
134
-static inline int z_erofs_init_workqueue(void)
146
+static struct kthread_worker *erofs_init_percpu_worker(int cpu)
135
147
 {
136
-	const unsigned int onlinecpus = num_possible_cpus();
148
+	struct kthread_worker *worker =
149
+		kthread_create_worker_on_cpu(cpu, 0, "erofs_worker/%u", cpu);
137
150
 
138
-	/*
139
-	 * no need to spawn too many threads, limiting threads could minimum
140
-	 * scheduling overhead, perhaps per-CPU threads should be better?
141
-	 */
142
-	z_erofs_workqueue = alloc_workqueue("erofs_unzipd",
143
-					    WQ_UNBOUND | WQ_HIGHPRI,
144
-					    onlinecpus + onlinecpus / 4);
145
-	return z_erofs_workqueue ? 0 : -ENOMEM;
151
+	if (IS_ERR(worker))
152
+		return worker;
153
+	if (IS_ENABLED(CONFIG_EROFS_FS_PCPU_KTHREAD_HIPRI))
154
+		sched_set_fifo_low(worker->task);
155
+	else
156
+		sched_set_normal(worker->task, 0);
157
+	return worker;
158
+}
159
+
160
+static int erofs_init_percpu_workers(void)
161
+{
162
+	struct kthread_worker *worker;
163
+	unsigned int cpu;
164
+
165
+	z_erofs_pcpu_workers = kcalloc(num_possible_cpus(),
166
+			sizeof(struct kthread_worker *), GFP_ATOMIC);
167
+	if (!z_erofs_pcpu_workers)
168
+		return -ENOMEM;
169
+
170
+	for_each_online_cpu(cpu) {	/* could miss cpu{off,on}line? */
171
+		worker = erofs_init_percpu_worker(cpu);
172
+		if (!IS_ERR(worker))
173
+			rcu_assign_pointer(z_erofs_pcpu_workers[cpu], worker);
174
+	}
175
+	return 0;
176
+}
177
+#else
178
+static inline void erofs_destroy_percpu_workers(void) {}
179
+static inline int erofs_init_percpu_workers(void) { return 0; }
180
+#endif
181
+
182
+#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_EROFS_FS_PCPU_KTHREAD)
183
+static DEFINE_SPINLOCK(z_erofs_pcpu_worker_lock);
184
+static enum cpuhp_state erofs_cpuhp_state;
185
+
186
+static int erofs_cpu_online(unsigned int cpu)
187
+{
188
+	struct kthread_worker *worker, *old;
189
+
190
+	worker = erofs_init_percpu_worker(cpu);
191
+	if (IS_ERR(worker))
192
+		return PTR_ERR(worker);
193
+
194
+	spin_lock(&z_erofs_pcpu_worker_lock);
195
+	old = rcu_dereference_protected(z_erofs_pcpu_workers[cpu],
196
+			lockdep_is_held(&z_erofs_pcpu_worker_lock));
197
+	if (!old)
198
+		rcu_assign_pointer(z_erofs_pcpu_workers[cpu], worker);
199
+	spin_unlock(&z_erofs_pcpu_worker_lock);
200
+	if (old)
201
+		kthread_destroy_worker(worker);
202
+	return 0;
203
+}
204
+
205
+static int erofs_cpu_offline(unsigned int cpu)
206
+{
207
+	struct kthread_worker *worker;
208
+
209
+	spin_lock(&z_erofs_pcpu_worker_lock);
210
+	worker = rcu_dereference_protected(z_erofs_pcpu_workers[cpu],
211
+			lockdep_is_held(&z_erofs_pcpu_worker_lock));
212
+	rcu_assign_pointer(z_erofs_pcpu_workers[cpu], NULL);
213
+	spin_unlock(&z_erofs_pcpu_worker_lock);
214
+
215
+	synchronize_rcu();
216
+	if (worker)
217
+		kthread_destroy_worker(worker);
218
+	return 0;
219
+}
220
+
221
+static int erofs_cpu_hotplug_init(void)
222
+{
223
+	int state;
224
+
225
+	state = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
226
+			"fs/erofs:online", erofs_cpu_online, erofs_cpu_offline);
227
+	if (state < 0)
228
+		return state;
229
+
230
+	erofs_cpuhp_state = state;
231
+	return 0;
232
+}
233
+
234
+static void erofs_cpu_hotplug_destroy(void)
235
+{
236
+	if (erofs_cpuhp_state)
237
+		cpuhp_remove_state_nocalls(erofs_cpuhp_state);
238
+}
239
+#else /* !CONFIG_HOTPLUG_CPU || !CONFIG_EROFS_FS_PCPU_KTHREAD */
240
+static inline int erofs_cpu_hotplug_init(void) { return 0; }
241
+static inline void erofs_cpu_hotplug_destroy(void) {}
242
+#endif
243
+
244
+void z_erofs_exit_zip_subsystem(void)
245
+{
246
+	erofs_cpu_hotplug_destroy();
247
+	erofs_destroy_percpu_workers();
248
+	destroy_workqueue(z_erofs_workqueue);
249
+	z_erofs_destroy_pcluster_pool();
146
250
 }
147
251
 
148
252
 int __init z_erofs_init_zip_subsystem(void)
..
..
@@ -150,10 +254,31 @@
150
254
 	int err = z_erofs_create_pcluster_pool();
151
255
 
152
256
 	if (err)
153
-		return err;
154
-	err = z_erofs_init_workqueue();
257
+		goto out_error_pcluster_pool;
258
+
259
+	z_erofs_workqueue = alloc_workqueue("erofs_worker",
260
+			WQ_UNBOUND | WQ_HIGHPRI, num_possible_cpus());
261
+	if (!z_erofs_workqueue) {
262
+		err = -ENOMEM;
263
+		goto out_error_workqueue_init;
264
+	}
265
+
266
+	err = erofs_init_percpu_workers();
155
267
 	if (err)
156
-		z_erofs_destroy_pcluster_pool();
268
+		goto out_error_pcpu_worker;
269
+
270
+	err = erofs_cpu_hotplug_init();
271
+	if (err < 0)
272
+		goto out_error_cpuhp_init;
273
+	return err;
274
+
275
+out_error_cpuhp_init:
276
+	erofs_destroy_percpu_workers();
277
+out_error_pcpu_worker:
278
+	destroy_workqueue(z_erofs_workqueue);
279
+out_error_workqueue_init:
280
+	z_erofs_destroy_pcluster_pool();
281
+out_error_pcluster_pool:
157
282
 	return err;
158
283
 }
159
284
 
..
..
@@ -719,9 +844,11 @@
719
844
 	tight &= (clt->mode >= COLLECT_PRIMARY_HOOKED &&
720
845
 		  clt->mode != COLLECT_PRIMARY_FOLLOWED_NOINPLACE);
721
846
 
722
-	cur = end - min_t(unsigned int, offset + end - map->m_la, end);
847
+	cur = end - min_t(erofs_off_t, offset + end - map->m_la, end);
723
848
 	if (!(map->m_flags & EROFS_MAP_MAPPED)) {
724
849
 		zero_user_segment(page, cur, end);
850
+		++spiltted;
851
+		tight = false;
725
852
 		goto next_part;
726
853
 	}
727
854
 
..
..
@@ -782,6 +909,12 @@
782
909
 }
783
910
 
784
911
 static void z_erofs_decompressqueue_work(struct work_struct *work);
912
+#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD
913
+static void z_erofs_decompressqueue_kthread_work(struct kthread_work *work)
914
+{
915
+	z_erofs_decompressqueue_work((struct work_struct *)work);
916
+}
917
+#endif
785
918
 static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io,
786
919
 				       bool sync, int bios)
787
920
 {
..
..
@@ -799,7 +932,22 @@
799
932
 		return;
800
933
 	/* Use workqueue and sync decompression for atomic contexts only */
801
934
 	if (in_atomic() || irqs_disabled()) {
935
+#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD
936
+		struct kthread_worker *worker;
937
+
938
+		rcu_read_lock();
939
+		worker = rcu_dereference(
940
+				z_erofs_pcpu_workers[raw_smp_processor_id()]);
941
+		if (!worker) {
942
+			INIT_WORK(&io->u.work, z_erofs_decompressqueue_work);
943
+			queue_work(z_erofs_workqueue, &io->u.work);
944
+		} else {
945
+			kthread_queue_work(worker, &io->u.kthread_work);
946
+		}
947
+		rcu_read_unlock();
948
+#else
802
949
 		queue_work(z_erofs_workqueue, &io->u.work);
950
+#endif
803
951
 		sbi->ctx.readahead_sync_decompress = true;
804
952
 		return;
805
953
 	}
..
..
@@ -1207,7 +1355,12 @@
1207
1355
 			*fg = true;
1208
1356
 			goto fg_out;
1209
1357
 		}
1358
+#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD
1359
+		kthread_init_work(&q->u.kthread_work,
1360
+				  z_erofs_decompressqueue_kthread_work);
1361
+#else
1210
1362
 		INIT_WORK(&q->u.work, z_erofs_decompressqueue_work);
1363
+#endif
1211
1364
 	} else {
1212
1365
 fg_out:
1213
1366
 		q = fgq;
..
..
@@ -1348,7 +1501,7 @@
1348
1501
 
1349
1502
 	/*
1350
1503
 	 * although background is preferred, no one is pending for submission.
1351
-	 * don't issue workqueue for decompression but drop it directly instead.
1504
+	 * don't issue decompression but drop it directly instead.
1352
1505
 	 */
1353
1506
 	if (!*force_fg && !nr_bios) {
1354
1507
 		kvfree(q[JQ_SUBMIT]);