forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/crypto/pcrypt.c
....@@ -1,21 +1,9 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * pcrypt - Parallel crypto wrapper.
34  *
45  * Copyright (C) 2009 secunet Security Networks AG
56  * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
6
- *
7
- * This program is free software; you can redistribute it and/or modify it
8
- * under the terms and conditions of the GNU General Public License,
9
- * version 2, as published by the Free Software Foundation.
10
- *
11
- * This program is distributed in the hope it will be useful, but WITHOUT
12
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
14
- * more details.
15
- *
16
- * You should have received a copy of the GNU General Public License along with
17
- * this program; if not, write to the Free Software Foundation, Inc.,
18
- * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
197  */
208 
219 #include <crypto/algapi.h>
....@@ -25,43 +13,18 @@
2513 #include <linux/init.h>
2614 #include <linux/module.h>
2715 #include <linux/slab.h>
28
-#include <linux/notifier.h>
2916 #include <linux/kobject.h>
3017 #include <linux/cpu.h>
3118 #include <crypto/pcrypt.h>
3219 
33
-struct padata_pcrypt {
34
-	struct padata_instance *pinst;
35
-	struct workqueue_struct *wq;
36
-
37
-	/*
38
-	 * Cpumask for callback CPUs. It should be
39
-	 * equal to serial cpumask of corresponding padata instance,
40
-	 * so it is updated when padata notifies us about serial
41
-	 * cpumask change.
42
-	 *
43
-	 * cb_cpumask is protected by RCU. This fact prevents us from
44
-	 * using cpumask_var_t directly because the actual type of
45
-	 * cpumsak_var_t depends on kernel configuration(particularly on
46
-	 * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration
47
-	 * cpumask_var_t may be either a pointer to the struct cpumask
48
-	 * or a variable allocated on the stack. Thus we can not safely use
49
-	 * cpumask_var_t with RCU operations such as rcu_assign_pointer or
50
-	 * rcu_dereference. So cpumask_var_t is wrapped with struct
51
-	 * pcrypt_cpumask which makes possible to use it with RCU.
52
-	 */
53
-	struct pcrypt_cpumask {
54
-		cpumask_var_t mask;
55
-	} *cb_cpumask;
56
-	struct notifier_block nblock;
57
-};
58
-
59
-static struct padata_pcrypt pencrypt;
60
-static struct padata_pcrypt pdecrypt;
20
+static struct padata_instance *pencrypt;
21
+static struct padata_instance *pdecrypt;
6122 static struct kset           *pcrypt_kset;
6223 
6324 struct pcrypt_instance_ctx {
6425 	struct crypto_aead_spawn spawn;
26
+	struct padata_shell *psenc;
27
+	struct padata_shell *psdec;
6528 	atomic_t tfm_count;
6629 };
6730 
....@@ -70,33 +33,10 @@
7033 	unsigned int cb_cpu;
7134 };
7235 
73
-static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu,
74
-			      struct padata_pcrypt *pcrypt)
36
+static inline struct pcrypt_instance_ctx *pcrypt_tfm_ictx(
37
+	struct crypto_aead *tfm)
7538 {
76
-	unsigned int cpu_index, cpu, i;
77
-	struct pcrypt_cpumask *cpumask;
78
-
79
-	cpu = *cb_cpu;
80
-
81
-	rcu_read_lock_bh();
82
-	cpumask = rcu_dereference_bh(pcrypt->cb_cpumask);
83
-	if (cpumask_test_cpu(cpu, cpumask->mask))
84
-			goto out;
85
-
86
-	if (!cpumask_weight(cpumask->mask))
87
-			goto out;
88
-
89
-	cpu_index = cpu % cpumask_weight(cpumask->mask);
90
-
91
-	cpu = cpumask_first(cpumask->mask);
92
-	for (i = 0; i < cpu_index; i++)
93
-		cpu = cpumask_next(cpu, cpumask->mask);
94
-
95
-	*cb_cpu = cpu;
96
-
97
-out:
98
-	rcu_read_unlock_bh();
99
-	return padata_do_parallel(pcrypt->pinst, padata, cpu);
39
+	return aead_instance_ctx(aead_alg_instance(tfm));
10040 }
10141 
10242 static int pcrypt_aead_setkey(struct crypto_aead *parent,
....@@ -158,6 +98,9 @@
15898 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
15999 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
160100 	u32 flags = aead_request_flags(req);
101
+	struct pcrypt_instance_ctx *ictx;
102
+
103
+	ictx = pcrypt_tfm_ictx(aead);
161104 
162105 	memset(padata, 0, sizeof(struct padata_priv));
163106 
....@@ -171,7 +114,7 @@
171114 			       req->cryptlen, req->iv);
172115 	aead_request_set_ad(creq, req->assoclen);
173116 
174
-	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt);
117
+	err = padata_do_parallel(ictx->psenc, padata, &ctx->cb_cpu);
175118 	if (!err)
176119 		return -EINPROGRESS;
177120 
....@@ -202,6 +145,9 @@
202145 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
203146 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
204147 	u32 flags = aead_request_flags(req);
148
+	struct pcrypt_instance_ctx *ictx;
149
+
150
+	ictx = pcrypt_tfm_ictx(aead);
205151 
206152 	memset(padata, 0, sizeof(struct padata_priv));
207153 
....@@ -215,7 +161,7 @@
215161 			       req->cryptlen, req->iv);
216162 	aead_request_set_ad(creq, req->assoclen);
217163 
218
-	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt);
164
+	err = padata_do_parallel(ictx->psdec, padata, &ctx->cb_cpu);
219165 	if (!err)
220166 		return -EINPROGRESS;
221167 
....@@ -262,6 +208,8 @@
262208 	struct pcrypt_instance_ctx *ctx = aead_instance_ctx(inst);
263209 
264210 	crypto_drop_aead(&ctx->spawn);
211
+	padata_free_shell(ctx->psdec);
212
+	padata_free_shell(ctx->psenc);
265213 	kfree(inst);
266214 }
267215 
....@@ -282,40 +230,40 @@
282230 }
283231 
284232 static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb,
285
-			      u32 type, u32 mask)
233
+			      struct crypto_attr_type *algt)
286234 {
287235 	struct pcrypt_instance_ctx *ctx;
288
-	struct crypto_attr_type *algt;
289236 	struct aead_instance *inst;
290237 	struct aead_alg *alg;
291
-	const char *name;
238
+	u32 mask = crypto_algt_inherited_mask(algt);
292239 	int err;
293
-
294
-	algt = crypto_get_attr_type(tb);
295
-	if (IS_ERR(algt))
296
-		return PTR_ERR(algt);
297
-
298
-	name = crypto_attr_alg_name(tb[1]);
299
-	if (IS_ERR(name))
300
-		return PTR_ERR(name);
301240 
302241 	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
303242 	if (!inst)
304243 		return -ENOMEM;
305244 
306
-	ctx = aead_instance_ctx(inst);
307
-	crypto_set_aead_spawn(&ctx->spawn, aead_crypto_instance(inst));
245
+	err = -ENOMEM;
308246 
309
-	err = crypto_grab_aead(&ctx->spawn, name, 0, 0);
247
+	ctx = aead_instance_ctx(inst);
248
+	ctx->psenc = padata_alloc_shell(pencrypt);
249
+	if (!ctx->psenc)
250
+		goto err_free_inst;
251
+
252
+	ctx->psdec = padata_alloc_shell(pdecrypt);
253
+	if (!ctx->psdec)
254
+		goto err_free_inst;
255
+
256
+	err = crypto_grab_aead(&ctx->spawn, aead_crypto_instance(inst),
257
+			       crypto_attr_alg_name(tb[1]), 0, mask);
310258 	if (err)
311
-		goto out_free_inst;
259
+		goto err_free_inst;
312260 
313261 	alg = crypto_spawn_aead_alg(&ctx->spawn);
314262 	err = pcrypt_init_instance(aead_crypto_instance(inst), &alg->base);
315263 	if (err)
316
-		goto out_drop_aead;
264
+		goto err_free_inst;
317265 
318
-	inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC;
266
+	inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC;
319267 
320268 	inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
321269 	inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
....@@ -333,17 +281,11 @@
333281 	inst->free = pcrypt_free;
334282 
335283 	err = aead_register_instance(tmpl, inst);
336
-	if (err)
337
-		goto out_drop_aead;
338
-
339
-out:
284
+	if (err) {
285
+err_free_inst:
286
+		pcrypt_free(inst);
287
+	}
340288 	return err;
341
-
342
-out_drop_aead:
343
-	crypto_drop_aead(&ctx->spawn);
344
-out_free_inst:
345
-	kfree(inst);
346
-	goto out;
347289 }
348290 
349291 static int pcrypt_create(struct crypto_template *tmpl, struct rtattr **tb)
....@@ -356,40 +298,10 @@
356298 
357299 	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
358300 	case CRYPTO_ALG_TYPE_AEAD:
359
-		return pcrypt_create_aead(tmpl, tb, algt->type, algt->mask);
301
+		return pcrypt_create_aead(tmpl, tb, algt);
360302 	}
361303 
362304 	return -EINVAL;
363
-}
364
-
365
-static int pcrypt_cpumask_change_notify(struct notifier_block *self,
366
-					unsigned long val, void *data)
367
-{
368
-	struct padata_pcrypt *pcrypt;
369
-	struct pcrypt_cpumask *new_mask, *old_mask;
370
-	struct padata_cpumask *cpumask = (struct padata_cpumask *)data;
371
-
372
-	if (!(val & PADATA_CPU_SERIAL))
373
-		return 0;
374
-
375
-	pcrypt = container_of(self, struct padata_pcrypt, nblock);
376
-	new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL);
377
-	if (!new_mask)
378
-		return -ENOMEM;
379
-	if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) {
380
-		kfree(new_mask);
381
-		return -ENOMEM;
382
-	}
383
-
384
-	old_mask = pcrypt->cb_cpumask;
385
-
386
-	cpumask_copy(new_mask->mask, cpumask->cbcpu);
387
-	rcu_assign_pointer(pcrypt->cb_cpumask, new_mask);
388
-	synchronize_rcu_bh();
389
-
390
-	free_cpumask_var(old_mask->mask);
391
-	kfree(old_mask);
392
-	return 0;
393305 }
394306 
395307 static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name)
....@@ -404,71 +316,19 @@
404316 	return ret;
405317 }
406318 
407
-static int pcrypt_init_padata(struct padata_pcrypt *pcrypt,
408
-			      const char *name)
319
+static int pcrypt_init_padata(struct padata_instance **pinst, const char *name)
409320 {
410321 	int ret = -ENOMEM;
411
-	struct pcrypt_cpumask *mask;
412322 
413
-	get_online_cpus();
323
+	*pinst = padata_alloc(name);
324
+	if (!*pinst)
325
+		return ret;
414326 
415
-	pcrypt->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE,
416
-				     1, name);
417
-	if (!pcrypt->wq)
418
-		goto err;
419
-
420
-	pcrypt->pinst = padata_alloc_possible(pcrypt->wq);
421
-	if (!pcrypt->pinst)
422
-		goto err_destroy_workqueue;
423
-
424
-	mask = kmalloc(sizeof(*mask), GFP_KERNEL);
425
-	if (!mask)
426
-		goto err_free_padata;
427
-	if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) {
428
-		kfree(mask);
429
-		goto err_free_padata;
430
-	}
431
-
432
-	cpumask_and(mask->mask, cpu_possible_mask, cpu_online_mask);
433
-	rcu_assign_pointer(pcrypt->cb_cpumask, mask);
434
-
435
-	pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify;
436
-	ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
327
+	ret = pcrypt_sysfs_add(*pinst, name);
437328 	if (ret)
438
-		goto err_free_cpumask;
439
-
440
-	ret = pcrypt_sysfs_add(pcrypt->pinst, name);
441
-	if (ret)
442
-		goto err_unregister_notifier;
443
-
444
-	put_online_cpus();
329
+		padata_free(*pinst);
445330 
446331 	return ret;
447
-
448
-err_unregister_notifier:
449
-	padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
450
-err_free_cpumask:
451
-	free_cpumask_var(mask->mask);
452
-	kfree(mask);
453
-err_free_padata:
454
-	padata_free(pcrypt->pinst);
455
-err_destroy_workqueue:
456
-	destroy_workqueue(pcrypt->wq);
457
-err:
458
-	put_online_cpus();
459
-
460
-	return ret;
461
-}
462
-
463
-static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt)
464
-{
465
-	free_cpumask_var(pcrypt->cb_cpumask->mask);
466
-	kfree(pcrypt->cb_cpumask);
467
-
468
-	padata_stop(pcrypt->pinst);
469
-	padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
470
-	destroy_workqueue(pcrypt->wq);
471
-	padata_free(pcrypt->pinst);
472332 }
473333 
474334 static struct crypto_template pcrypt_tmpl = {
....@@ -493,13 +353,10 @@
493353 	if (err)
494354 		goto err_deinit_pencrypt;
495355 
496
-	padata_start(pencrypt.pinst);
497
-	padata_start(pdecrypt.pinst);
498
-
499356 	return crypto_register_template(&pcrypt_tmpl);
500357 
501358 err_deinit_pencrypt:
502
-	pcrypt_fini_padata(&pencrypt);
359
+	padata_free(pencrypt);
503360 err_unreg_kset:
504361 	kset_unregister(pcrypt_kset);
505362 err:
....@@ -510,13 +367,13 @@
510367 {
511368 	crypto_unregister_template(&pcrypt_tmpl);
512369 
513
-	pcrypt_fini_padata(&pencrypt);
514
-	pcrypt_fini_padata(&pdecrypt);
370
+	padata_free(pencrypt);
371
+	padata_free(pdecrypt);
515372 
516373 	kset_unregister(pcrypt_kset);
517374 }
518375 
519
-module_init(pcrypt_init);
376
+subsys_initcall(pcrypt_init);
520377 module_exit(pcrypt_exit);
521378 
522379 MODULE_LICENSE("GPL");