hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d
kernel/drivers/crypto/caam/caampkc.c
....@@ -1,7 +1,9 @@
1
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
12 /*
23 * caam - Freescale FSL CAAM support for Public Key Cryptography
34 *
45 * Copyright 2016 Freescale Semiconductor, Inc.
6
+ * Copyright 2018-2019 NXP
57 *
68 * There is no Shared Descriptor for PKC so that the Job Descriptor must carry
79 * all the desired key parameters, input and output pointers.
....@@ -15,19 +17,36 @@
1517 #include "sg_sw_sec4.h"
1618 #include "caampkc.h"
1719
18
-#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + sizeof(struct rsa_pub_pdb))
20
+#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + SIZEOF_RSA_PUB_PDB)
1921 #define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \
20
- sizeof(struct rsa_priv_f1_pdb))
22
+ SIZEOF_RSA_PRIV_F1_PDB)
2123 #define DESC_RSA_PRIV_F2_LEN (2 * CAAM_CMD_SZ + \
22
- sizeof(struct rsa_priv_f2_pdb))
24
+ SIZEOF_RSA_PRIV_F2_PDB)
2325 #define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \
24
- sizeof(struct rsa_priv_f3_pdb))
26
+ SIZEOF_RSA_PRIV_F3_PDB)
27
+#define CAAM_RSA_MAX_INPUT_SIZE 512 /* for a 4096-bit modulus */
28
+
29
+/* buffer filled with zeros, used for padding */
30
+static u8 *zero_buffer;
31
+
32
+/*
33
+ * variable used to avoid double free of resources in case
34
+ * algorithm registration was unsuccessful
35
+ */
36
+static bool init_done;
37
+
38
+struct caam_akcipher_alg {
39
+ struct akcipher_alg akcipher;
40
+ bool registered;
41
+};
2542
2643 static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc,
2744 struct akcipher_request *req)
2845 {
46
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
47
+
2948 dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE);
30
- dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
49
+ dma_unmap_sg(dev, req_ctx->fixup_src, edesc->src_nents, DMA_TO_DEVICE);
3150
3251 if (edesc->sec4_sg_bytes)
3352 dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes,
....@@ -98,74 +117,82 @@
98117 static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context)
99118 {
100119 struct akcipher_request *req = context;
120
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
121
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
101122 struct rsa_edesc *edesc;
123
+ int ecode = 0;
124
+ bool has_bklog;
102125
103126 if (err)
104
- caam_jr_strstatus(dev, err);
127
+ ecode = caam_jr_strstatus(dev, err);
105128
106
- edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
129
+ edesc = req_ctx->edesc;
130
+ has_bklog = edesc->bklog;
107131
108132 rsa_pub_unmap(dev, edesc, req);
109133 rsa_io_unmap(dev, edesc, req);
110134 kfree(edesc);
111135
112
- akcipher_request_complete(req, err);
136
+ /*
137
+ * If no backlog flag, the completion of the request is done
138
+ * by CAAM, not crypto engine.
139
+ */
140
+ if (!has_bklog)
141
+ akcipher_request_complete(req, ecode);
142
+ else
143
+ crypto_finalize_akcipher_request(jrp->engine, req, ecode);
113144 }
114145
115
-static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err,
116
- void *context)
146
+static void rsa_priv_f_done(struct device *dev, u32 *desc, u32 err,
147
+ void *context)
117148 {
118149 struct akcipher_request *req = context;
150
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
151
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
152
+ struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
153
+ struct caam_rsa_key *key = &ctx->key;
154
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
119155 struct rsa_edesc *edesc;
156
+ int ecode = 0;
157
+ bool has_bklog;
120158
121159 if (err)
122
- caam_jr_strstatus(dev, err);
160
+ ecode = caam_jr_strstatus(dev, err);
123161
124
- edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
162
+ edesc = req_ctx->edesc;
163
+ has_bklog = edesc->bklog;
125164
126
- rsa_priv_f1_unmap(dev, edesc, req);
165
+ switch (key->priv_form) {
166
+ case FORM1:
167
+ rsa_priv_f1_unmap(dev, edesc, req);
168
+ break;
169
+ case FORM2:
170
+ rsa_priv_f2_unmap(dev, edesc, req);
171
+ break;
172
+ case FORM3:
173
+ rsa_priv_f3_unmap(dev, edesc, req);
174
+ }
175
+
127176 rsa_io_unmap(dev, edesc, req);
128177 kfree(edesc);
129178
130
- akcipher_request_complete(req, err);
179
+ /*
180
+ * If no backlog flag, the completion of the request is done
181
+ * by CAAM, not crypto engine.
182
+ */
183
+ if (!has_bklog)
184
+ akcipher_request_complete(req, ecode);
185
+ else
186
+ crypto_finalize_akcipher_request(jrp->engine, req, ecode);
131187 }
132188
133
-static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err,
134
- void *context)
135
-{
136
- struct akcipher_request *req = context;
137
- struct rsa_edesc *edesc;
138
-
139
- if (err)
140
- caam_jr_strstatus(dev, err);
141
-
142
- edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
143
-
144
- rsa_priv_f2_unmap(dev, edesc, req);
145
- rsa_io_unmap(dev, edesc, req);
146
- kfree(edesc);
147
-
148
- akcipher_request_complete(req, err);
149
-}
150
-
151
-static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
152
- void *context)
153
-{
154
- struct akcipher_request *req = context;
155
- struct rsa_edesc *edesc;
156
-
157
- if (err)
158
- caam_jr_strstatus(dev, err);
159
-
160
- edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
161
-
162
- rsa_priv_f3_unmap(dev, edesc, req);
163
- rsa_io_unmap(dev, edesc, req);
164
- kfree(edesc);
165
-
166
- akcipher_request_complete(req, err);
167
-}
168
-
189
+/**
190
+ * Count leading zeros, need it to strip, from a given scatterlist
191
+ *
192
+ * @sgl : scatterlist to count zeros from
193
+ * @nbytes: number of zeros, in bytes, to strip
194
+ * @flags : operation flags
195
+ */
169196 static int caam_rsa_count_leading_zeros(struct scatterlist *sgl,
170197 unsigned int nbytes,
171198 unsigned int flags)
....@@ -185,7 +212,8 @@
185212 lzeros = 0;
186213 len = 0;
187214 while (nbytes > 0) {
188
- while (len && !*buff) {
215
+ /* do not strip more than given bytes */
216
+ while (len && !*buff && lzeros < nbytes) {
189217 lzeros++;
190218 len--;
191219 buff++;
....@@ -216,29 +244,67 @@
216244 struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
217245 struct device *dev = ctx->dev;
218246 struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
247
+ struct caam_rsa_key *key = &ctx->key;
219248 struct rsa_edesc *edesc;
220249 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
221250 GFP_KERNEL : GFP_ATOMIC;
222251 int sg_flags = (flags == GFP_ATOMIC) ? SG_MITER_ATOMIC : 0;
223
- int sgc;
224252 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
225253 int src_nents, dst_nents;
254
+ int mapped_src_nents, mapped_dst_nents;
255
+ unsigned int diff_size = 0;
226256 int lzeros;
227257
228
- lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len, sg_flags);
229
- if (lzeros < 0)
230
- return ERR_PTR(lzeros);
258
+ if (req->src_len > key->n_sz) {
259
+ /*
260
+ * strip leading zeros and
261
+ * return the number of zeros to skip
262
+ */
263
+ lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len -
264
+ key->n_sz, sg_flags);
265
+ if (lzeros < 0)
266
+ return ERR_PTR(lzeros);
231267
232
- req->src_len -= lzeros;
233
- req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros);
268
+ req_ctx->fixup_src = scatterwalk_ffwd(req_ctx->src, req->src,
269
+ lzeros);
270
+ req_ctx->fixup_src_len = req->src_len - lzeros;
271
+ } else {
272
+ /*
273
+ * input src is less then n key modulus,
274
+ * so there will be zero padding
275
+ */
276
+ diff_size = key->n_sz - req->src_len;
277
+ req_ctx->fixup_src = req->src;
278
+ req_ctx->fixup_src_len = req->src_len;
279
+ }
234280
235
- src_nents = sg_nents_for_len(req->src, req->src_len);
281
+ src_nents = sg_nents_for_len(req_ctx->fixup_src,
282
+ req_ctx->fixup_src_len);
236283 dst_nents = sg_nents_for_len(req->dst, req->dst_len);
237284
238
- if (src_nents > 1)
239
- sec4_sg_len = src_nents;
240
- if (dst_nents > 1)
241
- sec4_sg_len += dst_nents;
285
+ mapped_src_nents = dma_map_sg(dev, req_ctx->fixup_src, src_nents,
286
+ DMA_TO_DEVICE);
287
+ if (unlikely(!mapped_src_nents)) {
288
+ dev_err(dev, "unable to map source\n");
289
+ return ERR_PTR(-ENOMEM);
290
+ }
291
+ mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
292
+ DMA_FROM_DEVICE);
293
+ if (unlikely(!mapped_dst_nents)) {
294
+ dev_err(dev, "unable to map destination\n");
295
+ goto src_fail;
296
+ }
297
+
298
+ if (!diff_size && mapped_src_nents == 1)
299
+ sec4_sg_len = 0; /* no need for an input hw s/g table */
300
+ else
301
+ sec4_sg_len = mapped_src_nents + !!diff_size;
302
+ sec4_sg_index = sec4_sg_len;
303
+
304
+ if (mapped_dst_nents > 1)
305
+ sec4_sg_len += pad_sg_nents(mapped_dst_nents);
306
+ else
307
+ sec4_sg_len = pad_sg_nents(sec4_sg_len);
242308
243309 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
244310
....@@ -246,37 +312,32 @@
246312 edesc = kzalloc(sizeof(*edesc) + desclen + sec4_sg_bytes,
247313 GFP_DMA | flags);
248314 if (!edesc)
249
- return ERR_PTR(-ENOMEM);
250
-
251
- sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
252
- if (unlikely(!sgc)) {
253
- dev_err(dev, "unable to map source\n");
254
- goto src_fail;
255
- }
256
-
257
- sgc = dma_map_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE);
258
- if (unlikely(!sgc)) {
259
- dev_err(dev, "unable to map destination\n");
260315 goto dst_fail;
261
- }
262316
263317 edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen;
318
+ if (diff_size)
319
+ dma_to_sec4_sg_one(edesc->sec4_sg, ctx->padding_dma, diff_size,
320
+ 0);
264321
265
- sec4_sg_index = 0;
266
- if (src_nents > 1) {
267
- sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
268
- sec4_sg_index += src_nents;
269
- }
270
- if (dst_nents > 1)
271
- sg_to_sec4_sg_last(req->dst, dst_nents,
322
+ if (sec4_sg_index)
323
+ sg_to_sec4_sg_last(req_ctx->fixup_src, req_ctx->fixup_src_len,
324
+ edesc->sec4_sg + !!diff_size, 0);
325
+
326
+ if (mapped_dst_nents > 1)
327
+ sg_to_sec4_sg_last(req->dst, req->dst_len,
272328 edesc->sec4_sg + sec4_sg_index, 0);
273329
274330 /* Save nents for later use in Job Descriptor */
275331 edesc->src_nents = src_nents;
276332 edesc->dst_nents = dst_nents;
277333
334
+ req_ctx->edesc = edesc;
335
+
278336 if (!sec4_sg_bytes)
279337 return edesc;
338
+
339
+ edesc->mapped_src_nents = mapped_src_nents;
340
+ edesc->mapped_dst_nents = mapped_dst_nents;
280341
281342 edesc->sec4_sg_dma = dma_map_single(dev, edesc->sec4_sg,
282343 sec4_sg_bytes, DMA_TO_DEVICE);
....@@ -287,21 +348,53 @@
287348
288349 edesc->sec4_sg_bytes = sec4_sg_bytes;
289350
351
+ print_hex_dump_debug("caampkc sec4_sg@" __stringify(__LINE__) ": ",
352
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
353
+ edesc->sec4_sg_bytes, 1);
354
+
290355 return edesc;
291356
292357 sec4_sg_fail:
293
- dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE);
294
-dst_fail:
295
- dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
296
-src_fail:
297358 kfree(edesc);
359
+dst_fail:
360
+ dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE);
361
+src_fail:
362
+ dma_unmap_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE);
298363 return ERR_PTR(-ENOMEM);
364
+}
365
+
366
+static int akcipher_do_one_req(struct crypto_engine *engine, void *areq)
367
+{
368
+ struct akcipher_request *req = container_of(areq,
369
+ struct akcipher_request,
370
+ base);
371
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
372
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
373
+ struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
374
+ struct device *jrdev = ctx->dev;
375
+ u32 *desc = req_ctx->edesc->hw_desc;
376
+ int ret;
377
+
378
+ req_ctx->edesc->bklog = true;
379
+
380
+ ret = caam_jr_enqueue(jrdev, desc, req_ctx->akcipher_op_done, req);
381
+
382
+ if (ret != -EINPROGRESS) {
383
+ rsa_pub_unmap(jrdev, req_ctx->edesc, req);
384
+ rsa_io_unmap(jrdev, req_ctx->edesc, req);
385
+ kfree(req_ctx->edesc);
386
+ } else {
387
+ ret = 0;
388
+ }
389
+
390
+ return ret;
299391 }
300392
301393 static int set_rsa_pub_pdb(struct akcipher_request *req,
302394 struct rsa_edesc *edesc)
303395 {
304396 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
397
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
305398 struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
306399 struct caam_rsa_key *key = &ctx->key;
307400 struct device *dev = ctx->dev;
....@@ -321,15 +414,15 @@
321414 return -ENOMEM;
322415 }
323416
324
- if (edesc->src_nents > 1) {
417
+ if (edesc->mapped_src_nents > 1) {
325418 pdb->sgf |= RSA_PDB_SGF_F;
326419 pdb->f_dma = edesc->sec4_sg_dma;
327
- sec4_sg_index += edesc->src_nents;
420
+ sec4_sg_index += edesc->mapped_src_nents;
328421 } else {
329
- pdb->f_dma = sg_dma_address(req->src);
422
+ pdb->f_dma = sg_dma_address(req_ctx->fixup_src);
330423 }
331424
332
- if (edesc->dst_nents > 1) {
425
+ if (edesc->mapped_dst_nents > 1) {
333426 pdb->sgf |= RSA_PDB_SGF_G;
334427 pdb->g_dma = edesc->sec4_sg_dma +
335428 sec4_sg_index * sizeof(struct sec4_sg_entry);
....@@ -338,7 +431,7 @@
338431 }
339432
340433 pdb->sgf |= (key->e_sz << RSA_PDB_E_SHIFT) | key->n_sz;
341
- pdb->f_len = req->src_len;
434
+ pdb->f_len = req_ctx->fixup_src_len;
342435
343436 return 0;
344437 }
....@@ -366,15 +459,18 @@
366459 return -ENOMEM;
367460 }
368461
369
- if (edesc->src_nents > 1) {
462
+ if (edesc->mapped_src_nents > 1) {
370463 pdb->sgf |= RSA_PRIV_PDB_SGF_G;
371464 pdb->g_dma = edesc->sec4_sg_dma;
372
- sec4_sg_index += edesc->src_nents;
465
+ sec4_sg_index += edesc->mapped_src_nents;
466
+
373467 } else {
374
- pdb->g_dma = sg_dma_address(req->src);
468
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
469
+
470
+ pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
375471 }
376472
377
- if (edesc->dst_nents > 1) {
473
+ if (edesc->mapped_dst_nents > 1) {
378474 pdb->sgf |= RSA_PRIV_PDB_SGF_F;
379475 pdb->f_dma = edesc->sec4_sg_dma +
380476 sec4_sg_index * sizeof(struct sec4_sg_entry);
....@@ -429,15 +525,17 @@
429525 goto unmap_tmp1;
430526 }
431527
432
- if (edesc->src_nents > 1) {
528
+ if (edesc->mapped_src_nents > 1) {
433529 pdb->sgf |= RSA_PRIV_PDB_SGF_G;
434530 pdb->g_dma = edesc->sec4_sg_dma;
435
- sec4_sg_index += edesc->src_nents;
531
+ sec4_sg_index += edesc->mapped_src_nents;
436532 } else {
437
- pdb->g_dma = sg_dma_address(req->src);
533
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
534
+
535
+ pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
438536 }
439537
440
- if (edesc->dst_nents > 1) {
538
+ if (edesc->mapped_dst_nents > 1) {
441539 pdb->sgf |= RSA_PRIV_PDB_SGF_F;
442540 pdb->f_dma = edesc->sec4_sg_dma +
443541 sec4_sg_index * sizeof(struct sec4_sg_entry);
....@@ -516,15 +614,17 @@
516614 goto unmap_tmp1;
517615 }
518616
519
- if (edesc->src_nents > 1) {
617
+ if (edesc->mapped_src_nents > 1) {
520618 pdb->sgf |= RSA_PRIV_PDB_SGF_G;
521619 pdb->g_dma = edesc->sec4_sg_dma;
522
- sec4_sg_index += edesc->src_nents;
620
+ sec4_sg_index += edesc->mapped_src_nents;
523621 } else {
524
- pdb->g_dma = sg_dma_address(req->src);
622
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
623
+
624
+ pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
525625 }
526626
527
- if (edesc->dst_nents > 1) {
627
+ if (edesc->mapped_dst_nents > 1) {
528628 pdb->sgf |= RSA_PRIV_PDB_SGF_F;
529629 pdb->f_dma = edesc->sec4_sg_dma +
530630 sec4_sg_index * sizeof(struct sec4_sg_entry);
....@@ -551,6 +651,53 @@
551651 dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE);
552652
553653 return -ENOMEM;
654
+}
655
+
656
+static int akcipher_enqueue_req(struct device *jrdev,
657
+ void (*cbk)(struct device *jrdev, u32 *desc,
658
+ u32 err, void *context),
659
+ struct akcipher_request *req)
660
+{
661
+ struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
662
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
663
+ struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
664
+ struct caam_rsa_key *key = &ctx->key;
665
+ struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
666
+ struct rsa_edesc *edesc = req_ctx->edesc;
667
+ u32 *desc = edesc->hw_desc;
668
+ int ret;
669
+
670
+ req_ctx->akcipher_op_done = cbk;
671
+ /*
672
+ * Only the backlog request are sent to crypto-engine since the others
673
+ * can be handled by CAAM, if free, especially since JR has up to 1024
674
+ * entries (more than the 10 entries from crypto-engine).
675
+ */
676
+ if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
677
+ ret = crypto_transfer_akcipher_request_to_engine(jrpriv->engine,
678
+ req);
679
+ else
680
+ ret = caam_jr_enqueue(jrdev, desc, cbk, req);
681
+
682
+ if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
683
+ switch (key->priv_form) {
684
+ case FORM1:
685
+ rsa_priv_f1_unmap(jrdev, edesc, req);
686
+ break;
687
+ case FORM2:
688
+ rsa_priv_f2_unmap(jrdev, edesc, req);
689
+ break;
690
+ case FORM3:
691
+ rsa_priv_f3_unmap(jrdev, edesc, req);
692
+ break;
693
+ default:
694
+ rsa_pub_unmap(jrdev, edesc, req);
695
+ }
696
+ rsa_io_unmap(jrdev, edesc, req);
697
+ kfree(edesc);
698
+ }
699
+
700
+ return ret;
554701 }
555702
556703 static int caam_rsa_enc(struct akcipher_request *req)
....@@ -584,11 +731,7 @@
584731 /* Initialize Job Descriptor */
585732 init_rsa_pub_desc(edesc->hw_desc, &edesc->pdb.pub);
586733
587
- ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_pub_done, req);
588
- if (!ret)
589
- return -EINPROGRESS;
590
-
591
- rsa_pub_unmap(jrdev, edesc, req);
734
+ return akcipher_enqueue_req(jrdev, rsa_pub_done, req);
592735
593736 init_fail:
594737 rsa_io_unmap(jrdev, edesc, req);
....@@ -617,11 +760,7 @@
617760 /* Initialize Job Descriptor */
618761 init_rsa_priv_f1_desc(edesc->hw_desc, &edesc->pdb.priv_f1);
619762
620
- ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f1_done, req);
621
- if (!ret)
622
- return -EINPROGRESS;
623
-
624
- rsa_priv_f1_unmap(jrdev, edesc, req);
763
+ return akcipher_enqueue_req(jrdev, rsa_priv_f_done, req);
625764
626765 init_fail:
627766 rsa_io_unmap(jrdev, edesc, req);
....@@ -650,11 +789,7 @@
650789 /* Initialize Job Descriptor */
651790 init_rsa_priv_f2_desc(edesc->hw_desc, &edesc->pdb.priv_f2);
652791
653
- ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f2_done, req);
654
- if (!ret)
655
- return -EINPROGRESS;
656
-
657
- rsa_priv_f2_unmap(jrdev, edesc, req);
792
+ return akcipher_enqueue_req(jrdev, rsa_priv_f_done, req);
658793
659794 init_fail:
660795 rsa_io_unmap(jrdev, edesc, req);
....@@ -683,11 +818,7 @@
683818 /* Initialize Job Descriptor */
684819 init_rsa_priv_f3_desc(edesc->hw_desc, &edesc->pdb.priv_f3);
685820
686
- ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f3_done, req);
687
- if (!ret)
688
- return -EINPROGRESS;
689
-
690
- rsa_priv_f3_unmap(jrdev, edesc, req);
821
+ return akcipher_enqueue_req(jrdev, rsa_priv_f_done, req);
691822
692823 init_fail:
693824 rsa_io_unmap(jrdev, edesc, req);
....@@ -723,14 +854,14 @@
723854
724855 static void caam_rsa_free_key(struct caam_rsa_key *key)
725856 {
726
- kzfree(key->d);
727
- kzfree(key->p);
728
- kzfree(key->q);
729
- kzfree(key->dp);
730
- kzfree(key->dq);
731
- kzfree(key->qinv);
732
- kzfree(key->tmp1);
733
- kzfree(key->tmp2);
857
+ kfree_sensitive(key->d);
858
+ kfree_sensitive(key->p);
859
+ kfree_sensitive(key->q);
860
+ kfree_sensitive(key->dp);
861
+ kfree_sensitive(key->dq);
862
+ kfree_sensitive(key->qinv);
863
+ kfree_sensitive(key->tmp1);
864
+ kfree_sensitive(key->tmp2);
734865 kfree(key->e);
735866 kfree(key->n);
736867 memset(key, 0, sizeof(*key));
....@@ -814,7 +945,7 @@
814945 return ret;
815946
816947 /* Copy key in DMA zone */
817
- rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA | GFP_KERNEL);
948
+ rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_DMA | GFP_KERNEL);
818949 if (!rsa_key->e)
819950 goto err;
820951
....@@ -835,8 +966,6 @@
835966
836967 rsa_key->e_sz = raw_key.e_sz;
837968 rsa_key->n_sz = raw_key.n_sz;
838
-
839
- memcpy(rsa_key->e, raw_key.e, raw_key.e_sz);
840969
841970 return 0;
842971 err:
....@@ -889,17 +1018,17 @@
8891018 return;
8901019
8911020 free_dq:
892
- kzfree(rsa_key->dq);
1021
+ kfree_sensitive(rsa_key->dq);
8931022 free_dp:
894
- kzfree(rsa_key->dp);
1023
+ kfree_sensitive(rsa_key->dp);
8951024 free_tmp2:
896
- kzfree(rsa_key->tmp2);
1025
+ kfree_sensitive(rsa_key->tmp2);
8971026 free_tmp1:
898
- kzfree(rsa_key->tmp1);
1027
+ kfree_sensitive(rsa_key->tmp1);
8991028 free_q:
900
- kzfree(rsa_key->q);
1029
+ kfree_sensitive(rsa_key->q);
9011030 free_p:
902
- kzfree(rsa_key->p);
1031
+ kfree_sensitive(rsa_key->p);
9031032 }
9041033
9051034 static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
....@@ -918,11 +1047,11 @@
9181047 return ret;
9191048
9201049 /* Copy key in DMA zone */
921
- rsa_key->d = kzalloc(raw_key.d_sz, GFP_DMA | GFP_KERNEL);
1050
+ rsa_key->d = kmemdup(raw_key.d, raw_key.d_sz, GFP_DMA | GFP_KERNEL);
9221051 if (!rsa_key->d)
9231052 goto err;
9241053
925
- rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA | GFP_KERNEL);
1054
+ rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_DMA | GFP_KERNEL);
9261055 if (!rsa_key->e)
9271056 goto err;
9281057
....@@ -944,9 +1073,6 @@
9441073 rsa_key->d_sz = raw_key.d_sz;
9451074 rsa_key->e_sz = raw_key.e_sz;
9461075 rsa_key->n_sz = raw_key.n_sz;
947
-
948
- memcpy(rsa_key->d, raw_key.d, raw_key.d_sz);
949
- memcpy(rsa_key->e, raw_key.e, raw_key.e_sz);
9501076
9511077 caam_rsa_set_priv_key_form(ctx, &raw_key);
9521078
....@@ -976,6 +1102,17 @@
9761102 return PTR_ERR(ctx->dev);
9771103 }
9781104
1105
+ ctx->padding_dma = dma_map_single(ctx->dev, zero_buffer,
1106
+ CAAM_RSA_MAX_INPUT_SIZE - 1,
1107
+ DMA_TO_DEVICE);
1108
+ if (dma_mapping_error(ctx->dev, ctx->padding_dma)) {
1109
+ dev_err(ctx->dev, "unable to map padding\n");
1110
+ caam_jr_free(ctx->dev);
1111
+ return -ENOMEM;
1112
+ }
1113
+
1114
+ ctx->enginectx.op.do_one_request = akcipher_do_one_req;
1115
+
9791116 return 0;
9801117 }
9811118
....@@ -985,90 +1122,90 @@
9851122 struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
9861123 struct caam_rsa_key *key = &ctx->key;
9871124
1125
+ dma_unmap_single(ctx->dev, ctx->padding_dma, CAAM_RSA_MAX_INPUT_SIZE -
1126
+ 1, DMA_TO_DEVICE);
9881127 caam_rsa_free_key(key);
9891128 caam_jr_free(ctx->dev);
9901129 }
9911130
992
-static struct akcipher_alg caam_rsa = {
993
- .encrypt = caam_rsa_enc,
994
- .decrypt = caam_rsa_dec,
995
- .sign = caam_rsa_dec,
996
- .verify = caam_rsa_enc,
997
- .set_pub_key = caam_rsa_set_pub_key,
998
- .set_priv_key = caam_rsa_set_priv_key,
999
- .max_size = caam_rsa_max_size,
1000
- .init = caam_rsa_init_tfm,
1001
- .exit = caam_rsa_exit_tfm,
1002
- .reqsize = sizeof(struct caam_rsa_req_ctx),
1003
- .base = {
1004
- .cra_name = "rsa",
1005
- .cra_driver_name = "rsa-caam",
1006
- .cra_priority = 3000,
1007
- .cra_module = THIS_MODULE,
1008
- .cra_ctxsize = sizeof(struct caam_rsa_ctx),
1009
- },
1131
+static struct caam_akcipher_alg caam_rsa = {
1132
+ .akcipher = {
1133
+ .encrypt = caam_rsa_enc,
1134
+ .decrypt = caam_rsa_dec,
1135
+ .set_pub_key = caam_rsa_set_pub_key,
1136
+ .set_priv_key = caam_rsa_set_priv_key,
1137
+ .max_size = caam_rsa_max_size,
1138
+ .init = caam_rsa_init_tfm,
1139
+ .exit = caam_rsa_exit_tfm,
1140
+ .reqsize = sizeof(struct caam_rsa_req_ctx),
1141
+ .base = {
1142
+ .cra_name = "rsa",
1143
+ .cra_driver_name = "rsa-caam",
1144
+ .cra_priority = 3000,
1145
+ .cra_module = THIS_MODULE,
1146
+ .cra_ctxsize = sizeof(struct caam_rsa_ctx),
1147
+ },
1148
+ }
10101149 };
10111150
10121151 /* Public Key Cryptography module initialization handler */
1013
-static int __init caam_pkc_init(void)
1152
+int caam_pkc_init(struct device *ctrldev)
10141153 {
1015
- struct device_node *dev_node;
1016
- struct platform_device *pdev;
1017
- struct device *ctrldev;
1018
- struct caam_drv_private *priv;
1019
- u32 cha_inst, pk_inst;
1154
+ struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
1155
+ u32 pk_inst, pkha;
10201156 int err;
1021
-
1022
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1023
- if (!dev_node) {
1024
- dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
1025
- if (!dev_node)
1026
- return -ENODEV;
1027
- }
1028
-
1029
- pdev = of_find_device_by_node(dev_node);
1030
- if (!pdev) {
1031
- of_node_put(dev_node);
1032
- return -ENODEV;
1033
- }
1034
-
1035
- ctrldev = &pdev->dev;
1036
- priv = dev_get_drvdata(ctrldev);
1037
- of_node_put(dev_node);
1038
-
1039
- /*
1040
- * If priv is NULL, it's probably because the caam driver wasn't
1041
- * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
1042
- */
1043
- if (!priv)
1044
- return -ENODEV;
1157
+ init_done = false;
10451158
10461159 /* Determine public key hardware accelerator presence. */
1047
- cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
1048
- pk_inst = (cha_inst & CHA_ID_LS_PK_MASK) >> CHA_ID_LS_PK_SHIFT;
1160
+ if (priv->era < 10) {
1161
+ pk_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
1162
+ CHA_ID_LS_PK_MASK) >> CHA_ID_LS_PK_SHIFT;
1163
+ } else {
1164
+ pkha = rd_reg32(&priv->ctrl->vreg.pkha);
1165
+ pk_inst = pkha & CHA_VER_NUM_MASK;
1166
+
1167
+ /*
1168
+ * Newer CAAMs support partially disabled functionality. If this is the
1169
+ * case, the number is non-zero, but this bit is set to indicate that
1170
+ * no encryption or decryption is supported. Only signing and verifying
1171
+ * is supported.
1172
+ */
1173
+ if (pkha & CHA_VER_MISC_PKHA_NO_CRYPT)
1174
+ pk_inst = 0;
1175
+ }
10491176
10501177 /* Do not register algorithms if PKHA is not present. */
10511178 if (!pk_inst)
1052
- return -ENODEV;
1179
+ return 0;
10531180
1054
- err = crypto_register_akcipher(&caam_rsa);
1055
- if (err)
1181
+ /* allocate zero buffer, used for padding input */
1182
+ zero_buffer = kzalloc(CAAM_RSA_MAX_INPUT_SIZE - 1, GFP_DMA |
1183
+ GFP_KERNEL);
1184
+ if (!zero_buffer)
1185
+ return -ENOMEM;
1186
+
1187
+ err = crypto_register_akcipher(&caam_rsa.akcipher);
1188
+
1189
+ if (err) {
1190
+ kfree(zero_buffer);
10561191 dev_warn(ctrldev, "%s alg registration failed\n",
1057
- caam_rsa.base.cra_driver_name);
1058
- else
1192
+ caam_rsa.akcipher.base.cra_driver_name);
1193
+ } else {
1194
+ init_done = true;
1195
+ caam_rsa.registered = true;
10591196 dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n");
1197
+ }
10601198
10611199 return err;
10621200 }
10631201
1064
-static void __exit caam_pkc_exit(void)
1202
+void caam_pkc_exit(void)
10651203 {
1066
- crypto_unregister_akcipher(&caam_rsa);
1204
+ if (!init_done)
1205
+ return;
1206
+
1207
+ if (caam_rsa.registered)
1208
+ crypto_unregister_akcipher(&caam_rsa.akcipher);
1209
+
1210
+ kfree(zero_buffer);
10671211 }
1068
-
1069
-module_init(caam_pkc_init);
1070
-module_exit(caam_pkc_exit);
1071
-
1072
-MODULE_LICENSE("Dual BSD/GPL");
1073
-MODULE_DESCRIPTION("FSL CAAM support for PKC functions of crypto API");
1074
-MODULE_AUTHOR("Freescale Semiconductor");