~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,7 +1,7 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Copyright (C) STMicroelectronics SA 2017
3	4	* Author: Fabien Dessenne <fabien.dessenne@st.com>
4		- * License terms: GNU General Public License (GPL), version 2
5	5	*/
6	6
7	7	#include <linux/clk.h>
..	..	@@ -15,10 +15,11 @@
15	15	#include <linux/reset.h>
16	16
17	17	#include <crypto/aes.h>
18		-#include <crypto/des.h>
	18	+#include <crypto/internal/des.h>
19	19	#include <crypto/engine.h>
20	20	#include <crypto/scatterwalk.h>
21	21	#include <crypto/internal/aead.h>
	22	+#include <crypto/internal/skcipher.h>
22	23
23	24	#define DRIVER_NAME "stm32-cryp"
24	25
..	..	@@ -36,7 +37,6 @@
36	37	/* Mode mask = bits [15..0] */
37	38	#define FLG_MODE_MASK GENMASK(15, 0)
38	39	/* Bit [31..16] status */
39		-#define FLG_CCM_PADDED_WA BIT(16)
40	40
41	41	/* Registers */
42	42	#define CRYP_CR 0x00000000
..	..	@@ -104,8 +104,6 @@
104	104	/* Misc */
105	105	#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32))
106	106	#define GCM_CTR_INIT 2
107		-#define _walked_in (cryp->in_walk.offset - cryp->in_sg->offset)
108		-#define _walked_out (cryp->out_walk.offset - cryp->out_sg->offset)
109	107	#define CRYP_AUTOSUSPEND_DELAY 50
110	108
111	109	struct stm32_cryp_caps {
..	..	@@ -117,7 +115,7 @@
117	115	struct crypto_engine_ctx enginectx;
118	116	struct stm32_cryp *cryp;
119	117	int keylen;
120		- u32 key[AES_KEYSIZE_256 / sizeof(u32)];
	118	+ __be32 key[AES_KEYSIZE_256 / sizeof(u32)];
121	119	unsigned long flags;
122	120	};
123	121
..	..	@@ -137,33 +135,22 @@
137	135
138	136	struct crypto_engine *engine;
139	137
140		- struct mutex lock; /* protects req / areq */
141		- struct ablkcipher_request *req;
	138	+ struct skcipher_request *req;
142	139	struct aead_request *areq;
143	140
144	141	size_t authsize;
145	142	size_t hw_blocksize;
146	143
147		- size_t total_in;
148		- size_t total_in_save;
149		- size_t total_out;
150		- size_t total_out_save;
	144	+ size_t payload_in;
	145	+ size_t header_in;
	146	+ size_t payload_out;
151	147
152		- struct scatterlist *in_sg;
153	148	struct scatterlist *out_sg;
154		- struct scatterlist *out_sg_save;
155		-
156		- struct scatterlist in_sgl;
157		- struct scatterlist out_sgl;
158		- bool sgs_copied;
159		-
160		- int in_sg_len;
161		- int out_sg_len;
162	149
163	150	struct scatter_walk in_walk;
164	151	struct scatter_walk out_walk;
165	152
166		- u32 last_ctr[4];
	153	+ __be32 last_ctr[4];
167	154	u32 gcm_ctr;
168	155	};
169	156
..	..	@@ -262,6 +249,7 @@
262	249	}
263	250
264	251	static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp);
	252	+static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err);
265	253
266	254	static struct stm32_cryp stm32_cryp_find_dev(struct stm32_cryp_ctx ctx)
267	255	{
..	..	@@ -283,114 +271,34 @@
283	271	return cryp;
284	272	}
285	273
286		-static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total,
287		- size_t align)
288		-{
289		- int len = 0;
290		-
291		- if (!total)
292		- return 0;
293		-
294		- if (!IS_ALIGNED(total, align))
295		- return -EINVAL;
296		-
297		- while (sg) {
298		- if (!IS_ALIGNED(sg->offset, sizeof(u32)))
299		- return -EINVAL;
300		-
301		- if (!IS_ALIGNED(sg->length, align))
302		- return -EINVAL;
303		-
304		- len += sg->length;
305		- sg = sg_next(sg);
306		- }
307		-
308		- if (len != total)
309		- return -EINVAL;
310		-
311		- return 0;
312		-}
313		-
314		-static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp)
315		-{
316		- int ret;
317		-
318		- ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in,
319		- cryp->hw_blocksize);
320		- if (ret)
321		- return ret;
322		-
323		- ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out,
324		- cryp->hw_blocksize);
325		-
326		- return ret;
327		-}
328		-
329		-static void sg_copy_buf(void buf, struct scatterlist sg,
330		- unsigned int start, unsigned int nbytes, int out)
331		-{
332		- struct scatter_walk walk;
333		-
334		- if (!nbytes)
335		- return;
336		-
337		- scatterwalk_start(&walk, sg);
338		- scatterwalk_advance(&walk, start);
339		- scatterwalk_copychunks(buf, &walk, nbytes, out);
340		- scatterwalk_done(&walk, out, 0);
341		-}
342		-
343		-static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp)
344		-{
345		- void buf_in, buf_out;
346		- int pages, total_in, total_out;
347		-
348		- if (!stm32_cryp_check_io_aligned(cryp)) {
349		- cryp->sgs_copied = 0;
350		- return 0;
351		- }
352		-
353		- total_in = ALIGN(cryp->total_in, cryp->hw_blocksize);
354		- pages = total_in ? get_order(total_in) : 1;
355		- buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
356		-
357		- total_out = ALIGN(cryp->total_out, cryp->hw_blocksize);
358		- pages = total_out ? get_order(total_out) : 1;
359		- buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
360		-
361		- if (!buf_in \|\| !buf_out) {
362		- dev_err(cryp->dev, "Can't allocate pages when unaligned\n");
363		- cryp->sgs_copied = 0;
364		- return -EFAULT;
365		- }
366		-
367		- sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0);
368		-
369		- sg_init_one(&cryp->in_sgl, buf_in, total_in);
370		- cryp->in_sg = &cryp->in_sgl;
371		- cryp->in_sg_len = 1;
372		-
373		- sg_init_one(&cryp->out_sgl, buf_out, total_out);
374		- cryp->out_sg_save = cryp->out_sg;
375		- cryp->out_sg = &cryp->out_sgl;
376		- cryp->out_sg_len = 1;
377		-
378		- cryp->sgs_copied = 1;
379		-
380		- return 0;
381		-}
382		-
383		-static void stm32_cryp_hw_write_iv(struct stm32_cryp cryp, u32 iv)
	274	+static void stm32_cryp_hw_write_iv(struct stm32_cryp cryp, __be32 iv)
384	275	{
385	276	if (!iv)
386	277	return;
387	278
388		- stm32_cryp_write(cryp, CRYP_IV0LR, cpu_to_be32(*iv++));
389		- stm32_cryp_write(cryp, CRYP_IV0RR, cpu_to_be32(*iv++));
	279	+ stm32_cryp_write(cryp, CRYP_IV0LR, be32_to_cpu(*iv++));
	280	+ stm32_cryp_write(cryp, CRYP_IV0RR, be32_to_cpu(*iv++));
390	281
391	282	if (is_aes(cryp)) {
392		- stm32_cryp_write(cryp, CRYP_IV1LR, cpu_to_be32(*iv++));
393		- stm32_cryp_write(cryp, CRYP_IV1RR, cpu_to_be32(*iv++));
	283	+ stm32_cryp_write(cryp, CRYP_IV1LR, be32_to_cpu(*iv++));
	284	+ stm32_cryp_write(cryp, CRYP_IV1RR, be32_to_cpu(*iv++));
	285	+ }
	286	+}
	287	+
	288	+static void stm32_cryp_get_iv(struct stm32_cryp *cryp)
	289	+{
	290	+ struct skcipher_request *req = cryp->req;
	291	+ __be32 tmp = (void )req->iv;
	292	+
	293	+ if (!tmp)
	294	+ return;
	295	+
	296	+ *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR));
	297	+ *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR));
	298	+
	299	+ if (is_aes(cryp)) {
	300	+ *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR));
	301	+ *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR));
394	302	}
395	303	}
396	304
..	..	@@ -400,13 +308,13 @@
400	308	int r_id;
401	309
402	310	if (is_des(c)) {
403		- stm32_cryp_write(c, CRYP_K1LR, cpu_to_be32(c->ctx->key[0]));
404		- stm32_cryp_write(c, CRYP_K1RR, cpu_to_be32(c->ctx->key[1]));
	311	+ stm32_cryp_write(c, CRYP_K1LR, be32_to_cpu(c->ctx->key[0]));
	312	+ stm32_cryp_write(c, CRYP_K1RR, be32_to_cpu(c->ctx->key[1]));
405	313	} else {
406	314	r_id = CRYP_K3RR;
407	315	for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4)
408	316	stm32_cryp_write(c, r_id,
409		- cpu_to_be32(c->ctx->key[i - 1]));
	317	+ be32_to_cpu(c->ctx->key[i - 1]));
410	318	}
411	319	}
412	320
..	..	@@ -452,7 +360,7 @@
452	360	static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg)
453	361	{
454	362	int ret;
455		- u32 iv[4];
	363	+ __be32 iv[4];
456	364
457	365	/* Phase 1 : init */
458	366	memcpy(iv, cryp->areq->iv, 12);
..	..	@@ -464,16 +372,100 @@
464	372
465	373	/* Wait for end of processing */
466	374	ret = stm32_cryp_wait_enable(cryp);
467		- if (ret)
	375	+ if (ret) {
468	376	dev_err(cryp->dev, "Timeout (gcm init)\n");
	377	+ return ret;
	378	+ }
469	379
470		- return ret;
	380	+ /* Prepare next phase */
	381	+ if (cryp->areq->assoclen) {
	382	+ cfg \|= CR_PH_HEADER;
	383	+ stm32_cryp_write(cryp, CRYP_CR, cfg);
	384	+ } else if (stm32_cryp_get_input_text_len(cryp)) {
	385	+ cfg \|= CR_PH_PAYLOAD;
	386	+ stm32_cryp_write(cryp, CRYP_CR, cfg);
	387	+ }
	388	+
	389	+ return 0;
	390	+}
	391	+
	392	+static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp)
	393	+{
	394	+ u32 cfg;
	395	+ int err;
	396	+
	397	+ /* Check if whole header written */
	398	+ if (!cryp->header_in) {
	399	+ /* Wait for completion */
	400	+ err = stm32_cryp_wait_busy(cryp);
	401	+ if (err) {
	402	+ dev_err(cryp->dev, "Timeout (gcm/ccm header)\n");
	403	+ stm32_cryp_write(cryp, CRYP_IMSCR, 0);
	404	+ stm32_cryp_finish_req(cryp, err);
	405	+ return;
	406	+ }
	407	+
	408	+ if (stm32_cryp_get_input_text_len(cryp)) {
	409	+ /* Phase 3 : payload */
	410	+ cfg = stm32_cryp_read(cryp, CRYP_CR);
	411	+ cfg &= ~CR_CRYPEN;
	412	+ stm32_cryp_write(cryp, CRYP_CR, cfg);
	413	+
	414	+ cfg &= ~CR_PH_MASK;
	415	+ cfg \|= CR_PH_PAYLOAD \| CR_CRYPEN;
	416	+ stm32_cryp_write(cryp, CRYP_CR, cfg);
	417	+ } else {
	418	+ /*
	419	+ * Phase 4 : tag.
	420	+ * Nothing to read, nothing to write, caller have to
	421	+ * end request
	422	+ */
	423	+ }
	424	+ }
	425	+}
	426	+
	427	+static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp)
	428	+{
	429	+ unsigned int i;
	430	+ size_t written;
	431	+ size_t len;
	432	+ u32 alen = cryp->areq->assoclen;
	433	+ u32 block[AES_BLOCK_32] = {0};
	434	+ u8 b8 = (u8 )block;
	435	+
	436	+ if (alen <= 65280) {
	437	+ /* Write first u32 of B1 */
	438	+ b8[0] = (alen >> 8) & 0xFF;
	439	+ b8[1] = alen & 0xFF;
	440	+ len = 2;
	441	+ } else {
	442	+ /* Build the two first u32 of B1 */
	443	+ b8[0] = 0xFF;
	444	+ b8[1] = 0xFE;
	445	+ b8[2] = (alen & 0xFF000000) >> 24;
	446	+ b8[3] = (alen & 0x00FF0000) >> 16;
	447	+ b8[4] = (alen & 0x0000FF00) >> 8;
	448	+ b8[5] = alen & 0x000000FF;
	449	+ len = 6;
	450	+ }
	451	+
	452	+ written = min_t(size_t, AES_BLOCK_SIZE - len, alen);
	453	+
	454	+ scatterwalk_copychunks((char *)block + len, &cryp->in_walk, written, 0);
	455	+ for (i = 0; i < AES_BLOCK_32; i++)
	456	+ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
	457	+
	458	+ cryp->header_in -= written;
	459	+
	460	+ stm32_crypt_gcmccm_end_header(cryp);
471	461	}
472	462
473	463	static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg)
474	464	{
475	465	int ret;
476		- u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
	466	+ u32 iv_32[AES_BLOCK_32], b0_32[AES_BLOCK_32];
	467	+ u8 iv = (u8 )iv_32, b0 = (u8 )b0_32;
	468	+ __be32 *bd;
477	469	u32 *d;
478	470	unsigned int i, textlen;
479	471
..	..	@@ -481,7 +473,7 @@
481	473	memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
482	474	memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
483	475	iv[AES_BLOCK_SIZE - 1] = 1;
484		- stm32_cryp_hw_write_iv(cryp, (u32 *)iv);
	476	+ stm32_cryp_hw_write_iv(cryp, (__be32 *)iv);
485	477
486	478	/* Build B0 */
487	479	memcpy(b0, iv, AES_BLOCK_SIZE);
..	..	@@ -501,19 +493,36 @@
501	493
502	494	/* Write B0 */
503	495	d = (u32 *)b0;
	496	+ bd = (__be32 *)b0;
504	497
505	498	for (i = 0; i < AES_BLOCK_32; i++) {
	499	+ u32 xd = d[i];
	500	+
506	501	if (!cryp->caps->padding_wa)
507		- d = cpu_to_be32(d);
508		- stm32_cryp_write(cryp, CRYP_DIN, *d++);
	502	+ xd = be32_to_cpu(bd[i]);
	503	+ stm32_cryp_write(cryp, CRYP_DIN, xd);
509	504	}
510	505
511	506	/* Wait for end of processing */
512	507	ret = stm32_cryp_wait_enable(cryp);
513		- if (ret)
	508	+ if (ret) {
514	509	dev_err(cryp->dev, "Timeout (ccm init)\n");
	510	+ return ret;
	511	+ }
515	512
516		- return ret;
	513	+ /* Prepare next phase */
	514	+ if (cryp->areq->assoclen) {
	515	+ cfg \|= CR_PH_HEADER \| CR_CRYPEN;
	516	+ stm32_cryp_write(cryp, CRYP_CR, cfg);
	517	+
	518	+ /* Write first (special) block (may move to next phase [payload]) */
	519	+ stm32_cryp_write_ccm_first_header(cryp);
	520	+ } else if (stm32_cryp_get_input_text_len(cryp)) {
	521	+ cfg \|= CR_PH_PAYLOAD;
	522	+ stm32_cryp_write(cryp, CRYP_CR, cfg);
	523	+ }
	524	+
	525	+ return 0;
517	526	}
518	527
519	528	static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
..	..	@@ -584,23 +593,13 @@
584	593	if (ret)
585	594	return ret;
586	595
587		- /* Phase 2 : header (authenticated data) */
588		- if (cryp->areq->assoclen) {
589		- cfg \|= CR_PH_HEADER;
590		- } else if (stm32_cryp_get_input_text_len(cryp)) {
591		- cfg \|= CR_PH_PAYLOAD;
592		- stm32_cryp_write(cryp, CRYP_CR, cfg);
593		- } else {
594		- cfg \|= CR_PH_INIT;
595		- }
596		-
597	596	break;
598	597
599	598	case CR_DES_CBC:
600	599	case CR_TDES_CBC:
601	600	case CR_AES_CBC:
602	601	case CR_AES_CTR:
603		- stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->req->info);
	602	+ stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv);
604	603	break;
605	604
606	605	default:
..	..	@@ -612,8 +611,6 @@
612	611
613	612	stm32_cryp_write(cryp, CRYP_CR, cfg);
614	613
615		- cryp->flags &= ~FLG_CCM_PADDED_WA;
616		-
617	614	return 0;
618	615	}
619	616
..	..	@@ -623,40 +620,17 @@
623	620	/* Phase 4 : output tag */
624	621	err = stm32_cryp_read_auth_tag(cryp);
625	622
626		- if (cryp->sgs_copied) {
627		- void buf_in, buf_out;
628		- int pages, len;
629		-
630		- buf_in = sg_virt(&cryp->in_sgl);
631		- buf_out = sg_virt(&cryp->out_sgl);
632		-
633		- sg_copy_buf(buf_out, cryp->out_sg_save, 0,
634		- cryp->total_out_save, 1);
635		-
636		- len = ALIGN(cryp->total_in_save, cryp->hw_blocksize);
637		- pages = len ? get_order(len) : 1;
638		- free_pages((unsigned long)buf_in, pages);
639		-
640		- len = ALIGN(cryp->total_out_save, cryp->hw_blocksize);
641		- pages = len ? get_order(len) : 1;
642		- free_pages((unsigned long)buf_out, pages);
643		- }
	623	+ if (!err && (!(is_gcm(cryp) \|\| is_ccm(cryp) \|\| is_ecb(cryp))))
	624	+ stm32_cryp_get_iv(cryp);
644	625
645	626	pm_runtime_mark_last_busy(cryp->dev);
646	627	pm_runtime_put_autosuspend(cryp->dev);
647	628
648		- if (is_gcm(cryp) \|\| is_ccm(cryp)) {
	629	+ if (is_gcm(cryp) \|\| is_ccm(cryp))
649	630	crypto_finalize_aead_request(cryp->engine, cryp->areq, err);
650		- cryp->areq = NULL;
651		- } else {
652		- crypto_finalize_ablkcipher_request(cryp->engine, cryp->req,
	631	+ else
	632	+ crypto_finalize_skcipher_request(cryp->engine, cryp->req,
653	633	err);
654		- cryp->req = NULL;
655		- }
656		-
657		- memset(cryp->ctx->key, 0, cryp->ctx->keylen);
658		-
659		- mutex_unlock(&cryp->lock);
660	634	}
661	635
662	636	static int stm32_cryp_cpu_start(struct stm32_cryp *cryp)
..	..	@@ -671,11 +645,11 @@
671	645	static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
672	646	void *areq);
673	647
674		-static int stm32_cryp_cra_init(struct crypto_tfm *tfm)
	648	+static int stm32_cryp_init_tfm(struct crypto_skcipher *tfm)
675	649	{
676		- struct stm32_cryp_ctx *ctx = crypto_tfm_ctx(tfm);
	650	+ struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
677	651
678		- tfm->crt_ablkcipher.reqsize = sizeof(struct stm32_cryp_reqctx);
	652	+ crypto_skcipher_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx));
679	653
680	654	ctx->enginectx.op.do_one_request = stm32_cryp_cipher_one_req;
681	655	ctx->enginectx.op.prepare_request = stm32_cryp_prepare_cipher_req;
..	..	@@ -700,11 +674,11 @@
700	674	return 0;
701	675	}
702	676
703		-static int stm32_cryp_crypt(struct ablkcipher_request *req, unsigned long mode)
	677	+static int stm32_cryp_crypt(struct skcipher_request *req, unsigned long mode)
704	678	{
705		- struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(
706		- crypto_ablkcipher_reqtfm(req));
707		- struct stm32_cryp_reqctx *rctx = ablkcipher_request_ctx(req);
	679	+ struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(
	680	+ crypto_skcipher_reqtfm(req));
	681	+ struct stm32_cryp_reqctx *rctx = skcipher_request_ctx(req);
708	682	struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);
709	683
710	684	if (!cryp)
..	..	@@ -712,7 +686,7 @@
712	686
713	687	rctx->mode = mode;
714	688
715		- return crypto_transfer_ablkcipher_request_to_engine(cryp->engine, req);
	689	+ return crypto_transfer_skcipher_request_to_engine(cryp->engine, req);
716	690	}
717	691
718	692	static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode)
..	..	@@ -729,10 +703,10 @@
729	703	return crypto_transfer_aead_request_to_engine(cryp->engine, req);
730	704	}
731	705
732		-static int stm32_cryp_setkey(struct crypto_ablkcipher tfm, const u8 key,
	706	+static int stm32_cryp_setkey(struct crypto_skcipher tfm, const u8 key,
733	707	unsigned int keylen)
734	708	{
735		- struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
	709	+ struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
736	710
737	711	memcpy(ctx->key, key, keylen);
738	712	ctx->keylen = keylen;
..	..	@@ -740,7 +714,7 @@
740	714	return 0;
741	715	}
742	716
743		-static int stm32_cryp_aes_setkey(struct crypto_ablkcipher tfm, const u8 key,
	717	+static int stm32_cryp_aes_setkey(struct crypto_skcipher tfm, const u8 key,
744	718	unsigned int keylen)
745	719	{
746	720	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
..	..	@@ -750,22 +724,18 @@
750	724	return stm32_cryp_setkey(tfm, key, keylen);
751	725	}
752	726
753		-static int stm32_cryp_des_setkey(struct crypto_ablkcipher tfm, const u8 key,
	727	+static int stm32_cryp_des_setkey(struct crypto_skcipher tfm, const u8 key,
754	728	unsigned int keylen)
755	729	{
756		- if (keylen != DES_KEY_SIZE)
757		- return -EINVAL;
758		- else
759		- return stm32_cryp_setkey(tfm, key, keylen);
	730	+ return verify_skcipher_des_key(tfm, key) ?:
	731	+ stm32_cryp_setkey(tfm, key, keylen);
760	732	}
761	733
762		-static int stm32_cryp_tdes_setkey(struct crypto_ablkcipher tfm, const u8 key,
	734	+static int stm32_cryp_tdes_setkey(struct crypto_skcipher tfm, const u8 key,
763	735	unsigned int keylen)
764	736	{
765		- if (keylen != (3 * DES_KEY_SIZE))
766		- return -EINVAL;
767		- else
768		- return stm32_cryp_setkey(tfm, key, keylen);
	737	+ return verify_skcipher_des3_key(tfm, key) ?:
	738	+ stm32_cryp_setkey(tfm, key, keylen);
769	739	}
770	740
771	741	static int stm32_cryp_aes_aead_setkey(struct crypto_aead tfm, const u8 key,
..	..	@@ -786,7 +756,20 @@
786	756	static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm,
787	757	unsigned int authsize)
788	758	{
789		- return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL;
	759	+ switch (authsize) {
	760	+ case 4:
	761	+ case 8:
	762	+ case 12:
	763	+ case 13:
	764	+ case 14:
	765	+ case 15:
	766	+ case 16:
	767	+ break;
	768	+ default:
	769	+ return -EINVAL;
	770	+ }
	771	+
	772	+ return 0;
790	773	}
791	774
792	775	static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm,
..	..	@@ -808,33 +791,63 @@
808	791	return 0;
809	792	}
810	793
811		-static int stm32_cryp_aes_ecb_encrypt(struct ablkcipher_request *req)
	794	+static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req)
812	795	{
	796	+ if (req->cryptlen % AES_BLOCK_SIZE)
	797	+ return -EINVAL;
	798	+
	799	+ if (req->cryptlen == 0)
	800	+ return 0;
	801	+
813	802	return stm32_cryp_crypt(req, FLG_AES \| FLG_ECB \| FLG_ENCRYPT);
814	803	}
815	804
816		-static int stm32_cryp_aes_ecb_decrypt(struct ablkcipher_request *req)
	805	+static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req)
817	806	{
	807	+ if (req->cryptlen % AES_BLOCK_SIZE)
	808	+ return -EINVAL;
	809	+
	810	+ if (req->cryptlen == 0)
	811	+ return 0;
	812	+
818	813	return stm32_cryp_crypt(req, FLG_AES \| FLG_ECB);
819	814	}
820	815
821		-static int stm32_cryp_aes_cbc_encrypt(struct ablkcipher_request *req)
	816	+static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req)
822	817	{
	818	+ if (req->cryptlen % AES_BLOCK_SIZE)
	819	+ return -EINVAL;
	820	+
	821	+ if (req->cryptlen == 0)
	822	+ return 0;
	823	+
823	824	return stm32_cryp_crypt(req, FLG_AES \| FLG_CBC \| FLG_ENCRYPT);
824	825	}
825	826
826		-static int stm32_cryp_aes_cbc_decrypt(struct ablkcipher_request *req)
	827	+static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req)
827	828	{
	829	+ if (req->cryptlen % AES_BLOCK_SIZE)
	830	+ return -EINVAL;
	831	+
	832	+ if (req->cryptlen == 0)
	833	+ return 0;
	834	+
828	835	return stm32_cryp_crypt(req, FLG_AES \| FLG_CBC);
829	836	}
830	837
831		-static int stm32_cryp_aes_ctr_encrypt(struct ablkcipher_request *req)
	838	+static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req)
832	839	{
	840	+ if (req->cryptlen == 0)
	841	+ return 0;
	842	+
833	843	return stm32_cryp_crypt(req, FLG_AES \| FLG_CTR \| FLG_ENCRYPT);
834	844	}
835	845
836		-static int stm32_cryp_aes_ctr_decrypt(struct ablkcipher_request *req)
	846	+static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req)
837	847	{
	848	+ if (req->cryptlen == 0)
	849	+ return 0;
	850	+
838	851	return stm32_cryp_crypt(req, FLG_AES \| FLG_CTR);
839	852	}
840	853
..	..	@@ -848,68 +861,138 @@
848	861	return stm32_cryp_aead_crypt(req, FLG_AES \| FLG_GCM);
849	862	}
850	863
	864	+static inline int crypto_ccm_check_iv(const u8 *iv)
	865	+{
	866	+ /* 2 <= L <= 8, so 1 <= L' <= 7. */
	867	+ if (iv[0] < 1 \|\| iv[0] > 7)
	868	+ return -EINVAL;
	869	+
	870	+ return 0;
	871	+}
	872	+
851	873	static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req)
852	874	{
	875	+ int err;
	876	+
	877	+ err = crypto_ccm_check_iv(req->iv);
	878	+ if (err)
	879	+ return err;
	880	+
853	881	return stm32_cryp_aead_crypt(req, FLG_AES \| FLG_CCM \| FLG_ENCRYPT);
854	882	}
855	883
856	884	static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req)
857	885	{
	886	+ int err;
	887	+
	888	+ err = crypto_ccm_check_iv(req->iv);
	889	+ if (err)
	890	+ return err;
	891	+
858	892	return stm32_cryp_aead_crypt(req, FLG_AES \| FLG_CCM);
859	893	}
860	894
861		-static int stm32_cryp_des_ecb_encrypt(struct ablkcipher_request *req)
	895	+static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req)
862	896	{
	897	+ if (req->cryptlen % DES_BLOCK_SIZE)
	898	+ return -EINVAL;
	899	+
	900	+ if (req->cryptlen == 0)
	901	+ return 0;
	902	+
863	903	return stm32_cryp_crypt(req, FLG_DES \| FLG_ECB \| FLG_ENCRYPT);
864	904	}
865	905
866		-static int stm32_cryp_des_ecb_decrypt(struct ablkcipher_request *req)
	906	+static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req)
867	907	{
	908	+ if (req->cryptlen % DES_BLOCK_SIZE)
	909	+ return -EINVAL;
	910	+
	911	+ if (req->cryptlen == 0)
	912	+ return 0;
	913	+
868	914	return stm32_cryp_crypt(req, FLG_DES \| FLG_ECB);
869	915	}
870	916
871		-static int stm32_cryp_des_cbc_encrypt(struct ablkcipher_request *req)
	917	+static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req)
872	918	{
	919	+ if (req->cryptlen % DES_BLOCK_SIZE)
	920	+ return -EINVAL;
	921	+
	922	+ if (req->cryptlen == 0)
	923	+ return 0;
	924	+
873	925	return stm32_cryp_crypt(req, FLG_DES \| FLG_CBC \| FLG_ENCRYPT);
874	926	}
875	927
876		-static int stm32_cryp_des_cbc_decrypt(struct ablkcipher_request *req)
	928	+static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req)
877	929	{
	930	+ if (req->cryptlen % DES_BLOCK_SIZE)
	931	+ return -EINVAL;
	932	+
	933	+ if (req->cryptlen == 0)
	934	+ return 0;
	935	+
878	936	return stm32_cryp_crypt(req, FLG_DES \| FLG_CBC);
879	937	}
880	938
881		-static int stm32_cryp_tdes_ecb_encrypt(struct ablkcipher_request *req)
	939	+static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req)
882	940	{
	941	+ if (req->cryptlen % DES_BLOCK_SIZE)
	942	+ return -EINVAL;
	943	+
	944	+ if (req->cryptlen == 0)
	945	+ return 0;
	946	+
883	947	return stm32_cryp_crypt(req, FLG_TDES \| FLG_ECB \| FLG_ENCRYPT);
884	948	}
885	949
886		-static int stm32_cryp_tdes_ecb_decrypt(struct ablkcipher_request *req)
	950	+static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req)
887	951	{
	952	+ if (req->cryptlen % DES_BLOCK_SIZE)
	953	+ return -EINVAL;
	954	+
	955	+ if (req->cryptlen == 0)
	956	+ return 0;
	957	+
888	958	return stm32_cryp_crypt(req, FLG_TDES \| FLG_ECB);
889	959	}
890	960
891		-static int stm32_cryp_tdes_cbc_encrypt(struct ablkcipher_request *req)
	961	+static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req)
892	962	{
	963	+ if (req->cryptlen % DES_BLOCK_SIZE)
	964	+ return -EINVAL;
	965	+
	966	+ if (req->cryptlen == 0)
	967	+ return 0;
	968	+
893	969	return stm32_cryp_crypt(req, FLG_TDES \| FLG_CBC \| FLG_ENCRYPT);
894	970	}
895	971
896		-static int stm32_cryp_tdes_cbc_decrypt(struct ablkcipher_request *req)
	972	+static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req)
897	973	{
	974	+ if (req->cryptlen % DES_BLOCK_SIZE)
	975	+ return -EINVAL;
	976	+
	977	+ if (req->cryptlen == 0)
	978	+ return 0;
	979	+
898	980	return stm32_cryp_crypt(req, FLG_TDES \| FLG_CBC);
899	981	}
900	982
901		-static int stm32_cryp_prepare_req(struct ablkcipher_request *req,
	983	+static int stm32_cryp_prepare_req(struct skcipher_request *req,
902	984	struct aead_request *areq)
903	985	{
904	986	struct stm32_cryp_ctx *ctx;
905	987	struct stm32_cryp *cryp;
906	988	struct stm32_cryp_reqctx *rctx;
	989	+ struct scatterlist *in_sg;
907	990	int ret;
908	991
909	992	if (!req && !areq)
910	993	return -EINVAL;
911	994
912		- ctx = req ? crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)) :
	995	+ ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) :
913	996	crypto_aead_ctx(crypto_aead_reqtfm(areq));
914	997
915	998	cryp = ctx->cryp;
..	..	@@ -917,9 +1000,7 @@
917	1000	if (!cryp)
918	1001	return -ENODEV;
919	1002
920		- mutex_lock(&cryp->lock);
921		-
922		- rctx = req ? ablkcipher_request_ctx(req) : aead_request_ctx(areq);
	1003	+ rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq);
923	1004	rctx->mode &= FLG_MODE_MASK;
924	1005
925	1006	ctx->cryp = cryp;
..	..	@@ -930,88 +1011,65 @@
930	1011
931	1012	if (req) {
932	1013	cryp->req = req;
933		- cryp->total_in = req->nbytes;
934		- cryp->total_out = cryp->total_in;
	1014	+ cryp->areq = NULL;
	1015	+ cryp->header_in = 0;
	1016	+ cryp->payload_in = req->cryptlen;
	1017	+ cryp->payload_out = req->cryptlen;
	1018	+ cryp->authsize = 0;
935	1019	} else {
936	1020	/*
937	1021	* Length of input and output data:
938	1022	* Encryption case:
939		- * INPUT = AssocData \|\| PlainText
	1023	+ * INPUT = AssocData \|\| PlainText
940	1024	* <- assoclen -> <- cryptlen ->
941		- * <------- total_in ----------->
942	1025	*
943		- * OUTPUT = AssocData \|\| CipherText \|\| AuthTag
944		- * <- assoclen -> <- cryptlen -> <- authsize ->
945		- * <---------------- total_out ----------------->
	1026	+ * OUTPUT = AssocData \|\| CipherText \|\| AuthTag
	1027	+ * <- assoclen -> <-- cryptlen --> <- authsize ->
946	1028	*
947	1029	* Decryption case:
948		- * INPUT = AssocData \|\| CipherText \|\| AuthTag
949		- * <- assoclen -> <--------- cryptlen --------->
950		- * <- authsize ->
951		- * <---------------- total_in ------------------>
	1030	+ * INPUT = AssocData \|\| CipherTex \|\| AuthTag
	1031	+ * <- assoclen ---> <---------- cryptlen ---------->
952	1032	*
953		- * OUTPUT = AssocData \|\| PlainText
954		- * <- assoclen -> <- crypten - authsize ->
955		- * <---------- total_out ----------------->
	1033	+ * OUTPUT = AssocData \|\| PlainText
	1034	+ * <- assoclen -> <- cryptlen - authsize ->
956	1035	*/
957	1036	cryp->areq = areq;
	1037	+ cryp->req = NULL;
958	1038	cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));
959		- cryp->total_in = areq->assoclen + areq->cryptlen;
960		- if (is_encrypt(cryp))
961		- /* Append auth tag to output */
962		- cryp->total_out = cryp->total_in + cryp->authsize;
963		- else
964		- /* No auth tag in output */
965		- cryp->total_out = cryp->total_in - cryp->authsize;
	1039	+ if (is_encrypt(cryp)) {
	1040	+ cryp->payload_in = areq->cryptlen;
	1041	+ cryp->header_in = areq->assoclen;
	1042	+ cryp->payload_out = areq->cryptlen;
	1043	+ } else {
	1044	+ cryp->payload_in = areq->cryptlen - cryp->authsize;
	1045	+ cryp->header_in = areq->assoclen;
	1046	+ cryp->payload_out = cryp->payload_in;
	1047	+ }
966	1048	}
967	1049
968		- cryp->total_in_save = cryp->total_in;
969		- cryp->total_out_save = cryp->total_out;
	1050	+ in_sg = req ? req->src : areq->src;
	1051	+ scatterwalk_start(&cryp->in_walk, in_sg);
970	1052
971		- cryp->in_sg = req ? req->src : areq->src;
972	1053	cryp->out_sg = req ? req->dst : areq->dst;
973		- cryp->out_sg_save = cryp->out_sg;
974		-
975		- cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in);
976		- if (cryp->in_sg_len < 0) {
977		- dev_err(cryp->dev, "Cannot get in_sg_len\n");
978		- ret = cryp->in_sg_len;
979		- goto out;
980		- }
981		-
982		- cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out);
983		- if (cryp->out_sg_len < 0) {
984		- dev_err(cryp->dev, "Cannot get out_sg_len\n");
985		- ret = cryp->out_sg_len;
986		- goto out;
987		- }
988		-
989		- ret = stm32_cryp_copy_sgs(cryp);
990		- if (ret)
991		- goto out;
992		-
993		- scatterwalk_start(&cryp->in_walk, cryp->in_sg);
994	1054	scatterwalk_start(&cryp->out_walk, cryp->out_sg);
995	1055
996	1056	if (is_gcm(cryp) \|\| is_ccm(cryp)) {
997	1057	/* In output, jump after assoc data */
998		- scatterwalk_advance(&cryp->out_walk, cryp->areq->assoclen);
999		- cryp->total_out -= cryp->areq->assoclen;
	1058	+ scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->areq->assoclen, 2);
1000	1059	}
1001	1060
1002		- ret = stm32_cryp_hw_init(cryp);
1003		-out:
1004		- if (ret)
1005		- mutex_unlock(&cryp->lock);
	1061	+ if (is_ctr(cryp))
	1062	+ memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr));
1006	1063
	1064	+ ret = stm32_cryp_hw_init(cryp);
1007	1065	return ret;
1008	1066	}
1009	1067
1010	1068	static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
1011	1069	void *areq)
1012	1070	{
1013		- struct ablkcipher_request *req = container_of(areq,
1014		- struct ablkcipher_request,
	1071	+ struct skcipher_request *req = container_of(areq,
	1072	+ struct skcipher_request,
1015	1073	base);
1016	1074
1017	1075	return stm32_cryp_prepare_req(req, NULL);
..	..	@@ -1019,11 +1077,11 @@
1019	1077
1020	1078	static int stm32_cryp_cipher_one_req(struct crypto_engine engine, void areq)
1021	1079	{
1022		- struct ablkcipher_request *req = container_of(areq,
1023		- struct ablkcipher_request,
	1080	+ struct skcipher_request *req = container_of(areq,
	1081	+ struct skcipher_request,
1024	1082	base);
1025		- struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(
1026		- crypto_ablkcipher_reqtfm(req));
	1083	+ struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(
	1084	+ crypto_skcipher_reqtfm(req));
1027	1085	struct stm32_cryp *cryp = ctx->cryp;
1028	1086
1029	1087	if (!cryp)
..	..	@@ -1050,8 +1108,7 @@
1050	1108	if (!cryp)
1051	1109	return -ENODEV;
1052	1110
1053		- if (unlikely(!cryp->areq->assoclen &&
1054		- !stm32_cryp_get_input_text_len(cryp))) {
	1111	+ if (unlikely(!cryp->payload_in && !cryp->header_in)) {
1055	1112	/* No input data to process: get tag and finish */
1056	1113	stm32_cryp_finish_req(cryp, 0);
1057	1114	return 0;
..	..	@@ -1060,43 +1117,10 @@
1060	1117	return stm32_cryp_cpu_start(cryp);
1061	1118	}
1062	1119
1063		-static u32 stm32_cryp_next_out(struct stm32_cryp cryp, u32 *dst,
1064		- unsigned int n)
1065		-{
1066		- scatterwalk_advance(&cryp->out_walk, n);
1067		-
1068		- if (unlikely(cryp->out_sg->length == _walked_out)) {
1069		- cryp->out_sg = sg_next(cryp->out_sg);
1070		- if (cryp->out_sg) {
1071		- scatterwalk_start(&cryp->out_walk, cryp->out_sg);
1072		- return (sg_virt(cryp->out_sg) + _walked_out);
1073		- }
1074		- }
1075		-
1076		- return (u32 )((u8 )dst + n);
1077		-}
1078		-
1079		-static u32 stm32_cryp_next_in(struct stm32_cryp cryp, u32 *src,
1080		- unsigned int n)
1081		-{
1082		- scatterwalk_advance(&cryp->in_walk, n);
1083		-
1084		- if (unlikely(cryp->in_sg->length == _walked_in)) {
1085		- cryp->in_sg = sg_next(cryp->in_sg);
1086		- if (cryp->in_sg) {
1087		- scatterwalk_start(&cryp->in_walk, cryp->in_sg);
1088		- return (sg_virt(cryp->in_sg) + _walked_in);
1089		- }
1090		- }
1091		-
1092		- return (u32 )((u8 )src + n);
1093		-}
1094		-
1095	1120	static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp)
1096	1121	{
1097		- u32 cfg, size_bit, *dst, d32;
1098		- u8 *d8;
1099		- unsigned int i, j;
	1122	+ u32 cfg, size_bit;
	1123	+ unsigned int i;
1100	1124	int ret = 0;
1101	1125
1102	1126	/* Update Config */
..	..	@@ -1113,31 +1137,34 @@
1113	1137	/* GCM: write aad and payload size (in bits) */
1114	1138	size_bit = cryp->areq->assoclen * 8;
1115	1139	if (cryp->caps->swap_final)
1116		- size_bit = cpu_to_be32(size_bit);
	1140	+ size_bit = (__force u32)cpu_to_be32(size_bit);
1117	1141
1118	1142	stm32_cryp_write(cryp, CRYP_DIN, 0);
1119	1143	stm32_cryp_write(cryp, CRYP_DIN, size_bit);
1120	1144
1121	1145	size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen :
1122		- cryp->areq->cryptlen - AES_BLOCK_SIZE;
	1146	+ cryp->areq->cryptlen - cryp->authsize;
1123	1147	size_bit *= 8;
1124	1148	if (cryp->caps->swap_final)
1125		- size_bit = cpu_to_be32(size_bit);
	1149	+ size_bit = (__force u32)cpu_to_be32(size_bit);
1126	1150
1127	1151	stm32_cryp_write(cryp, CRYP_DIN, 0);
1128	1152	stm32_cryp_write(cryp, CRYP_DIN, size_bit);
1129	1153	} else {
1130	1154	/* CCM: write CTR0 */
1131		- u8 iv[AES_BLOCK_SIZE];
1132		- u32 iv32 = (u32 )iv;
	1155	+ u32 iv32[AES_BLOCK_32];
	1156	+ u8 iv = (u8 )iv32;
	1157	+ __be32 biv = (__be32 )iv32;
1133	1158
1134	1159	memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
1135	1160	memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
1136	1161
1137	1162	for (i = 0; i < AES_BLOCK_32; i++) {
	1163	+ u32 xiv = iv32[i];
	1164	+
1138	1165	if (!cryp->caps->padding_wa)
1139		- iv32 = cpu_to_be32(iv32);
1140		- stm32_cryp_write(cryp, CRYP_DIN, *iv32++);
	1166	+ xiv = be32_to_cpu(biv[i]);
	1167	+ stm32_cryp_write(cryp, CRYP_DIN, xiv);
1141	1168	}
1142	1169	}
1143	1170
..	..	@@ -1149,39 +1176,18 @@
1149	1176	}
1150	1177
1151	1178	if (is_encrypt(cryp)) {
	1179	+ u32 out_tag[AES_BLOCK_32];
	1180	+
1152	1181	/* Get and write tag */
1153		- dst = sg_virt(cryp->out_sg) + _walked_out;
	1182	+ for (i = 0; i < AES_BLOCK_32; i++)
	1183	+ out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1154	1184
1155		- for (i = 0; i < AES_BLOCK_32; i++) {
1156		- if (cryp->total_out >= sizeof(u32)) {
1157		- /* Read a full u32 */
1158		- *dst = stm32_cryp_read(cryp, CRYP_DOUT);
1159		-
1160		- dst = stm32_cryp_next_out(cryp, dst,
1161		- sizeof(u32));
1162		- cryp->total_out -= sizeof(u32);
1163		- } else if (!cryp->total_out) {
1164		- /* Empty fifo out (data from input padding) */
1165		- stm32_cryp_read(cryp, CRYP_DOUT);
1166		- } else {
1167		- /* Read less than an u32 */
1168		- d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1169		- d8 = (u8 *)&d32;
1170		-
1171		- for (j = 0; j < cryp->total_out; j++) {
1172		- ((u8 )dst) = *(d8++);
1173		- dst = stm32_cryp_next_out(cryp, dst, 1);
1174		- }
1175		- cryp->total_out = 0;
1176		- }
1177		- }
	1185	+ scatterwalk_copychunks(out_tag, &cryp->out_walk, cryp->authsize, 1);
1178	1186	} else {
1179	1187	/* Get and check tag */
1180	1188	u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32];
1181	1189
1182		- scatterwalk_map_and_copy(in_tag, cryp->in_sg,
1183		- cryp->total_in_save - cryp->authsize,
1184		- cryp->authsize, 0);
	1190	+ scatterwalk_copychunks(in_tag, &cryp->in_walk, cryp->authsize, 0);
1185	1191
1186	1192	for (i = 0; i < AES_BLOCK_32; i++)
1187	1193	out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT);
..	..	@@ -1201,115 +1207,59 @@
1201	1207	{
1202	1208	u32 cr;
1203	1209
1204		- if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) {
1205		- cryp->last_ctr[3] = 0;
1206		- cryp->last_ctr[2]++;
1207		- if (!cryp->last_ctr[2]) {
1208		- cryp->last_ctr[1]++;
1209		- if (!cryp->last_ctr[1])
1210		- cryp->last_ctr[0]++;
1211		- }
	1210	+ if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) {
	1211	+ /*
	1212	+ * In this case, we need to increment manually the ctr counter,
	1213	+ * as HW doesn't handle the U32 carry.
	1214	+ */
	1215	+ crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr));
1212	1216
1213	1217	cr = stm32_cryp_read(cryp, CRYP_CR);
1214	1218	stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN);
1215	1219
1216		- stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->last_ctr);
	1220	+ stm32_cryp_hw_write_iv(cryp, cryp->last_ctr);
1217	1221
1218	1222	stm32_cryp_write(cryp, CRYP_CR, cr);
1219	1223	}
1220	1224
1221		- cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR);
1222		- cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR);
1223		- cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR);
1224		- cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR);
	1225	+ /* The IV registers are BE */
	1226	+ cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR));
	1227	+ cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR));
	1228	+ cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR));
	1229	+ cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR));
1225	1230	}
1226	1231
1227		-static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
	1232	+static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
1228	1233	{
1229		- unsigned int i, j;
1230		- u32 d32, *dst;
1231		- u8 *d8;
1232		- size_t tag_size;
	1234	+ unsigned int i;
	1235	+ u32 block[AES_BLOCK_32];
1233	1236
1234		- /* Do no read tag now (if any) */
1235		- if (is_encrypt(cryp) && (is_gcm(cryp) \|\| is_ccm(cryp)))
1236		- tag_size = cryp->authsize;
1237		- else
1238		- tag_size = 0;
	1237	+ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
	1238	+ block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1239	1239
1240		- dst = sg_virt(cryp->out_sg) + _walked_out;
1241		-
1242		- for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
1243		- if (likely(cryp->total_out - tag_size >= sizeof(u32))) {
1244		- /* Read a full u32 */
1245		- *dst = stm32_cryp_read(cryp, CRYP_DOUT);
1246		-
1247		- dst = stm32_cryp_next_out(cryp, dst, sizeof(u32));
1248		- cryp->total_out -= sizeof(u32);
1249		- } else if (cryp->total_out == tag_size) {
1250		- /* Empty fifo out (data from input padding) */
1251		- d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1252		- } else {
1253		- /* Read less than an u32 */
1254		- d32 = stm32_cryp_read(cryp, CRYP_DOUT);
1255		- d8 = (u8 *)&d32;
1256		-
1257		- for (j = 0; j < cryp->total_out - tag_size; j++) {
1258		- ((u8 )dst) = *(d8++);
1259		- dst = stm32_cryp_next_out(cryp, dst, 1);
1260		- }
1261		- cryp->total_out = tag_size;
1262		- }
1263		- }
1264		-
1265		- return !(cryp->total_out - tag_size) \|\| !cryp->total_in;
	1240	+ scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
	1241	+ cryp->payload_out), 1);
	1242	+ cryp->payload_out -= min_t(size_t, cryp->hw_blocksize,
	1243	+ cryp->payload_out);
1266	1244	}
1267	1245
1268	1246	static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp)
1269	1247	{
1270		- unsigned int i, j;
1271		- u32 *src;
1272		- u8 d8[4];
1273		- size_t tag_size;
	1248	+ unsigned int i;
	1249	+ u32 block[AES_BLOCK_32] = {0};
1274	1250
1275		- /* Do no write tag (if any) */
1276		- if (is_decrypt(cryp) && (is_gcm(cryp) \|\| is_ccm(cryp)))
1277		- tag_size = cryp->authsize;
1278		- else
1279		- tag_size = 0;
	1251	+ scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize,
	1252	+ cryp->payload_in), 0);
	1253	+ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
	1254	+ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
1280	1255
1281		- src = sg_virt(cryp->in_sg) + _walked_in;
1282		-
1283		- for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
1284		- if (likely(cryp->total_in - tag_size >= sizeof(u32))) {
1285		- /* Write a full u32 */
1286		- stm32_cryp_write(cryp, CRYP_DIN, *src);
1287		-
1288		- src = stm32_cryp_next_in(cryp, src, sizeof(u32));
1289		- cryp->total_in -= sizeof(u32);
1290		- } else if (cryp->total_in == tag_size) {
1291		- /* Write padding data */
1292		- stm32_cryp_write(cryp, CRYP_DIN, 0);
1293		- } else {
1294		- /* Write less than an u32 */
1295		- memset(d8, 0, sizeof(u32));
1296		- for (j = 0; j < cryp->total_in - tag_size; j++) {
1297		- d8[j] = ((u8 )src);
1298		- src = stm32_cryp_next_in(cryp, src, 1);
1299		- }
1300		-
1301		- stm32_cryp_write(cryp, CRYP_DIN, (u32 )d8);
1302		- cryp->total_in = tag_size;
1303		- }
1304		- }
	1256	+ cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in);
1305	1257	}
1306	1258
1307	1259	static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp)
1308	1260	{
1309	1261	int err;
1310		- u32 cfg, tmp[AES_BLOCK_32];
1311		- size_t total_in_ori = cryp->total_in;
1312		- struct scatterlist *out_sg_ori = cryp->out_sg;
	1262	+ u32 cfg, block[AES_BLOCK_32] = {0};
1313	1263	unsigned int i;
1314	1264
1315	1265	/* 'Special workaround' procedure described in the datasheet */
..	..	@@ -1334,18 +1284,25 @@
1334	1284
1335	1285	/* b) pad and write the last block */
1336	1286	stm32_cryp_irq_write_block(cryp);
1337		- cryp->total_in = total_in_ori;
	1287	+ /* wait end of process */
1338	1288	err = stm32_cryp_wait_output(cryp);
1339	1289	if (err) {
1340		- dev_err(cryp->dev, "Timeout (write gcm header)\n");
	1290	+ dev_err(cryp->dev, "Timeout (write gcm last data)\n");
1341	1291	return stm32_cryp_finish_req(cryp, err);
1342	1292	}
1343	1293
1344	1294	/* c) get and store encrypted data */
1345		- stm32_cryp_irq_read_data(cryp);
1346		- scatterwalk_map_and_copy(tmp, out_sg_ori,
1347		- cryp->total_in_save - total_in_ori,
1348		- total_in_ori, 0);
	1295	+ /*
	1296	+ * Same code as stm32_cryp_irq_read_data(), but we want to store
	1297	+ * block value
	1298	+ */
	1299	+ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
	1300	+ block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
	1301	+
	1302	+ scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
	1303	+ cryp->payload_out), 1);
	1304	+ cryp->payload_out -= min_t(size_t, cryp->hw_blocksize,
	1305	+ cryp->payload_out);
1349	1306
1350	1307	/* d) change mode back to AES GCM */
1351	1308	cfg &= ~CR_ALGO_MASK;
..	..	@@ -1358,19 +1315,13 @@
1358	1315	stm32_cryp_write(cryp, CRYP_CR, cfg);
1359	1316
1360	1317	/* f) write padded data */
1361		- for (i = 0; i < AES_BLOCK_32; i++) {
1362		- if (cryp->total_in)
1363		- stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
1364		- else
1365		- stm32_cryp_write(cryp, CRYP_DIN, 0);
1366		-
1367		- cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
1368		- }
	1318	+ for (i = 0; i < AES_BLOCK_32; i++)
	1319	+ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
1369	1320
1370	1321	/* g) Empty fifo out */
1371	1322	err = stm32_cryp_wait_output(cryp);
1372	1323	if (err) {
1373		- dev_err(cryp->dev, "Timeout (write gcm header)\n");
	1324	+ dev_err(cryp->dev, "Timeout (write gcm padded data)\n");
1374	1325	return stm32_cryp_finish_req(cryp, err);
1375	1326	}
1376	1327
..	..	@@ -1383,16 +1334,14 @@
1383	1334
1384	1335	static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp)
1385	1336	{
1386		- u32 cfg, payload_bytes;
	1337	+ u32 cfg;
1387	1338
1388	1339	/* disable ip, set NPBLB and reneable ip */
1389	1340	cfg = stm32_cryp_read(cryp, CRYP_CR);
1390	1341	cfg &= ~CR_CRYPEN;
1391	1342	stm32_cryp_write(cryp, CRYP_CR, cfg);
1392	1343
1393		- payload_bytes = is_decrypt(cryp) ? cryp->total_in - cryp->authsize :
1394		- cryp->total_in;
1395		- cfg \|= (cryp->hw_blocksize - payload_bytes) << CR_NBPBL_SHIFT;
	1344	+ cfg \|= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT;
1396	1345	cfg \|= CR_CRYPEN;
1397	1346	stm32_cryp_write(cryp, CRYP_CR, cfg);
1398	1347	}
..	..	@@ -1401,13 +1350,11 @@
1401	1350	{
1402	1351	int err = 0;
1403	1352	u32 cfg, iv1tmp;
1404		- u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32], tmp[AES_BLOCK_32];
1405		- size_t last_total_out, total_in_ori = cryp->total_in;
1406		- struct scatterlist *out_sg_ori = cryp->out_sg;
	1353	+ u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32];
	1354	+ u32 block[AES_BLOCK_32] = {0};
1407	1355	unsigned int i;
1408	1356
1409	1357	/* 'Special workaround' procedure described in the datasheet */
1410		- cryp->flags \|= FLG_CCM_PADDED_WA;
1411	1358
1412	1359	/* a) disable ip */
1413	1360	stm32_cryp_write(cryp, CRYP_IMSCR, 0);
..	..	@@ -1437,7 +1384,7 @@
1437	1384
1438	1385	/* b) pad and write the last block */
1439	1386	stm32_cryp_irq_write_block(cryp);
1440		- cryp->total_in = total_in_ori;
	1387	+ /* wait end of process */
1441	1388	err = stm32_cryp_wait_output(cryp);
1442	1389	if (err) {
1443	1390	dev_err(cryp->dev, "Timeout (wite ccm padded data)\n");
..	..	@@ -1445,13 +1392,16 @@
1445	1392	}
1446	1393
1447	1394	/* c) get and store decrypted data */
1448		- last_total_out = cryp->total_out;
1449		- stm32_cryp_irq_read_data(cryp);
	1395	+ /*
	1396	+ * Same code as stm32_cryp_irq_read_data(), but we want to store
	1397	+ * block value
	1398	+ */
	1399	+ for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++)
	1400	+ block[i] = stm32_cryp_read(cryp, CRYP_DOUT);
1450	1401
1451		- memset(tmp, 0, sizeof(tmp));
1452		- scatterwalk_map_and_copy(tmp, out_sg_ori,
1453		- cryp->total_out_save - last_total_out,
1454		- last_total_out, 0);
	1402	+ scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize,
	1403	+ cryp->payload_out), 1);
	1404	+ cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out);
1455	1405
1456	1406	/* d) Load again CRYP_CSGCMCCMxR */
1457	1407	for (i = 0; i < ARRAY_SIZE(cstmp2); i++)
..	..	@@ -1468,10 +1418,10 @@
1468	1418	stm32_cryp_write(cryp, CRYP_CR, cfg);
1469	1419
1470	1420	/* g) XOR and write padded data */
1471		- for (i = 0; i < ARRAY_SIZE(tmp); i++) {
1472		- tmp[i] ^= cstmp1[i];
1473		- tmp[i] ^= cstmp2[i];
1474		- stm32_cryp_write(cryp, CRYP_DIN, tmp[i]);
	1421	+ for (i = 0; i < ARRAY_SIZE(block); i++) {
	1422	+ block[i] ^= cstmp1[i];
	1423	+ block[i] ^= cstmp2[i];
	1424	+ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
1475	1425	}
1476	1426
1477	1427	/* h) wait for completion */
..	..	@@ -1485,30 +1435,34 @@
1485	1435
1486	1436	static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)
1487	1437	{
1488		- if (unlikely(!cryp->total_in)) {
	1438	+ if (unlikely(!cryp->payload_in)) {
1489	1439	dev_warn(cryp->dev, "No more data to process\n");
1490	1440	return;
1491	1441	}
1492	1442
1493		- if (unlikely(cryp->total_in < AES_BLOCK_SIZE &&
	1443	+ if (unlikely(cryp->payload_in < AES_BLOCK_SIZE &&
1494	1444	(stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) &&
1495	1445	is_encrypt(cryp))) {
1496	1446	/* Padding for AES GCM encryption */
1497		- if (cryp->caps->padding_wa)
	1447	+ if (cryp->caps->padding_wa) {
1498	1448	/* Special case 1 */
1499		- return stm32_cryp_irq_write_gcm_padded_data(cryp);
	1449	+ stm32_cryp_irq_write_gcm_padded_data(cryp);
	1450	+ return;
	1451	+ }
1500	1452
1501	1453	/* Setting padding bytes (NBBLB) */
1502	1454	stm32_cryp_irq_set_npblb(cryp);
1503	1455	}
1504	1456
1505		- if (unlikely((cryp->total_in - cryp->authsize < AES_BLOCK_SIZE) &&
	1457	+ if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) &&
1506	1458	(stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) &&
1507	1459	is_decrypt(cryp))) {
1508	1460	/* Padding for AES CCM decryption */
1509		- if (cryp->caps->padding_wa)
	1461	+ if (cryp->caps->padding_wa) {
1510	1462	/* Special case 2 */
1511		- return stm32_cryp_irq_write_ccm_padded_data(cryp);
	1463	+ stm32_cryp_irq_write_ccm_padded_data(cryp);
	1464	+ return;
	1465	+ }
1512	1466
1513	1467	/* Setting padding bytes (NBBLB) */
1514	1468	stm32_cryp_irq_set_npblb(cryp);
..	..	@@ -1520,191 +1474,59 @@
1520	1474	stm32_cryp_irq_write_block(cryp);
1521	1475	}
1522	1476
1523		-static void stm32_cryp_irq_write_gcm_header(struct stm32_cryp *cryp)
	1477	+static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp)
1524	1478	{
1525		- int err;
1526		- unsigned int i, j;
1527		- u32 cfg, *src;
	1479	+ unsigned int i;
	1480	+ u32 block[AES_BLOCK_32] = {0};
	1481	+ size_t written;
1528	1482
1529		- src = sg_virt(cryp->in_sg) + _walked_in;
	1483	+ written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in);
1530	1484
1531		- for (i = 0; i < AES_BLOCK_32; i++) {
1532		- stm32_cryp_write(cryp, CRYP_DIN, *src);
	1485	+ scatterwalk_copychunks(block, &cryp->in_walk, written, 0);
	1486	+ for (i = 0; i < AES_BLOCK_32; i++)
	1487	+ stm32_cryp_write(cryp, CRYP_DIN, block[i]);
1533	1488
1534		- src = stm32_cryp_next_in(cryp, src, sizeof(u32));
1535		- cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in);
	1489	+ cryp->header_in -= written;
1536	1490
1537		- /* Check if whole header written */
1538		- if ((cryp->total_in_save - cryp->total_in) ==
1539		- cryp->areq->assoclen) {
1540		- /* Write padding if needed */
1541		- for (j = i + 1; j < AES_BLOCK_32; j++)
1542		- stm32_cryp_write(cryp, CRYP_DIN, 0);
1543		-
1544		- /* Wait for completion */
1545		- err = stm32_cryp_wait_busy(cryp);
1546		- if (err) {
1547		- dev_err(cryp->dev, "Timeout (gcm header)\n");
1548		- return stm32_cryp_finish_req(cryp, err);
1549		- }
1550		-
1551		- if (stm32_cryp_get_input_text_len(cryp)) {
1552		- /* Phase 3 : payload */
1553		- cfg = stm32_cryp_read(cryp, CRYP_CR);
1554		- cfg &= ~CR_CRYPEN;
1555		- stm32_cryp_write(cryp, CRYP_CR, cfg);
1556		-
1557		- cfg &= ~CR_PH_MASK;
1558		- cfg \|= CR_PH_PAYLOAD;
1559		- cfg \|= CR_CRYPEN;
1560		- stm32_cryp_write(cryp, CRYP_CR, cfg);
1561		- } else {
1562		- /* Phase 4 : tag */
1563		- stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1564		- stm32_cryp_finish_req(cryp, 0);
1565		- }
1566		-
1567		- break;
1568		- }
1569		-
1570		- if (!cryp->total_in)
1571		- break;
1572		- }
1573		-}
1574		-
1575		-static void stm32_cryp_irq_write_ccm_header(struct stm32_cryp *cryp)
1576		-{
1577		- int err;
1578		- unsigned int i = 0, j, k;
1579		- u32 alen, cfg, *src;
1580		- u8 d8[4];
1581		-
1582		- src = sg_virt(cryp->in_sg) + _walked_in;
1583		- alen = cryp->areq->assoclen;
1584		-
1585		- if (!_walked_in) {
1586		- if (cryp->areq->assoclen <= 65280) {
1587		- /* Write first u32 of B1 */
1588		- d8[0] = (alen >> 8) & 0xFF;
1589		- d8[1] = alen & 0xFF;
1590		- d8[2] = ((u8 )src);
1591		- src = stm32_cryp_next_in(cryp, src, 1);
1592		- d8[3] = ((u8 )src);
1593		- src = stm32_cryp_next_in(cryp, src, 1);
1594		-
1595		- stm32_cryp_write(cryp, CRYP_DIN, (u32 )d8);
1596		- i++;
1597		-
1598		- cryp->total_in -= min_t(size_t, 2, cryp->total_in);
1599		- } else {
1600		- /* Build the two first u32 of B1 */
1601		- d8[0] = 0xFF;
1602		- d8[1] = 0xFE;
1603		- d8[2] = alen & 0xFF000000;
1604		- d8[3] = alen & 0x00FF0000;
1605		-
1606		- stm32_cryp_write(cryp, CRYP_DIN, (u32 )d8);
1607		- i++;
1608		-
1609		- d8[0] = alen & 0x0000FF00;
1610		- d8[1] = alen & 0x000000FF;
1611		- d8[2] = ((u8 )src);
1612		- src = stm32_cryp_next_in(cryp, src, 1);
1613		- d8[3] = ((u8 )src);
1614		- src = stm32_cryp_next_in(cryp, src, 1);
1615		-
1616		- stm32_cryp_write(cryp, CRYP_DIN, (u32 )d8);
1617		- i++;
1618		-
1619		- cryp->total_in -= min_t(size_t, 2, cryp->total_in);
1620		- }
1621		- }
1622		-
1623		- /* Write next u32 */
1624		- for (; i < AES_BLOCK_32; i++) {
1625		- /* Build an u32 */
1626		- memset(d8, 0, sizeof(u32));
1627		- for (k = 0; k < sizeof(u32); k++) {
1628		- d8[k] = ((u8 )src);
1629		- src = stm32_cryp_next_in(cryp, src, 1);
1630		-
1631		- cryp->total_in -= min_t(size_t, 1, cryp->total_in);
1632		- if ((cryp->total_in_save - cryp->total_in) == alen)
1633		- break;
1634		- }
1635		-
1636		- stm32_cryp_write(cryp, CRYP_DIN, (u32 )d8);
1637		-
1638		- if ((cryp->total_in_save - cryp->total_in) == alen) {
1639		- /* Write padding if needed */
1640		- for (j = i + 1; j < AES_BLOCK_32; j++)
1641		- stm32_cryp_write(cryp, CRYP_DIN, 0);
1642		-
1643		- /* Wait for completion */
1644		- err = stm32_cryp_wait_busy(cryp);
1645		- if (err) {
1646		- dev_err(cryp->dev, "Timeout (ccm header)\n");
1647		- return stm32_cryp_finish_req(cryp, err);
1648		- }
1649		-
1650		- if (stm32_cryp_get_input_text_len(cryp)) {
1651		- /* Phase 3 : payload */
1652		- cfg = stm32_cryp_read(cryp, CRYP_CR);
1653		- cfg &= ~CR_CRYPEN;
1654		- stm32_cryp_write(cryp, CRYP_CR, cfg);
1655		-
1656		- cfg &= ~CR_PH_MASK;
1657		- cfg \|= CR_PH_PAYLOAD;
1658		- cfg \|= CR_CRYPEN;
1659		- stm32_cryp_write(cryp, CRYP_CR, cfg);
1660		- } else {
1661		- /* Phase 4 : tag */
1662		- stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1663		- stm32_cryp_finish_req(cryp, 0);
1664		- }
1665		-
1666		- break;
1667		- }
1668		- }
	1491	+ stm32_crypt_gcmccm_end_header(cryp);
1669	1492	}
1670	1493
1671	1494	static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg)
1672	1495	{
1673	1496	struct stm32_cryp *cryp = arg;
1674	1497	u32 ph;
	1498	+ u32 it_mask = stm32_cryp_read(cryp, CRYP_IMSCR);
1675	1499
1676	1500	if (cryp->irq_status & MISR_OUT)
1677	1501	/* Output FIFO IRQ: read data */
1678		- if (unlikely(stm32_cryp_irq_read_data(cryp))) {
1679		- /* All bytes processed, finish */
1680		- stm32_cryp_write(cryp, CRYP_IMSCR, 0);
1681		- stm32_cryp_finish_req(cryp, 0);
1682		- return IRQ_HANDLED;
1683		- }
	1502	+ stm32_cryp_irq_read_data(cryp);
1684	1503
1685	1504	if (cryp->irq_status & MISR_IN) {
1686		- if (is_gcm(cryp)) {
	1505	+ if (is_gcm(cryp) \|\| is_ccm(cryp)) {
1687	1506	ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
1688	1507	if (unlikely(ph == CR_PH_HEADER))
1689	1508	/* Write Header */
1690		- stm32_cryp_irq_write_gcm_header(cryp);
	1509	+ stm32_cryp_irq_write_gcmccm_header(cryp);
1691	1510	else
1692	1511	/* Input FIFO IRQ: write data */
1693	1512	stm32_cryp_irq_write_data(cryp);
1694		- cryp->gcm_ctr++;
1695		- } else if (is_ccm(cryp)) {
1696		- ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK;
1697		- if (unlikely(ph == CR_PH_HEADER))
1698		- /* Write Header */
1699		- stm32_cryp_irq_write_ccm_header(cryp);
1700		- else
1701		- /* Input FIFO IRQ: write data */
1702		- stm32_cryp_irq_write_data(cryp);
	1513	+ if (is_gcm(cryp))
	1514	+ cryp->gcm_ctr++;
1703	1515	} else {
1704	1516	/* Input FIFO IRQ: write data */
1705	1517	stm32_cryp_irq_write_data(cryp);
1706	1518	}
1707	1519	}
	1520	+
	1521	+ /* Mask useless interrupts */
	1522	+ if (!cryp->payload_in && !cryp->header_in)
	1523	+ it_mask &= ~IMSCR_IN;
	1524	+ if (!cryp->payload_out)
	1525	+ it_mask &= ~IMSCR_OUT;
	1526	+ stm32_cryp_write(cryp, CRYP_IMSCR, it_mask);
	1527	+
	1528	+ if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out)
	1529	+ stm32_cryp_finish_req(cryp, 0);
1708	1530
1709	1531	return IRQ_HANDLED;
1710	1532	}
..	..	@@ -1718,150 +1540,129 @@
1718	1540	return IRQ_WAKE_THREAD;
1719	1541	}
1720	1542
1721		-static struct crypto_alg crypto_algs[] = {
	1543	+static struct skcipher_alg crypto_algs[] = {
1722	1544	{
1723		- .cra_name = "ecb(aes)",
1724		- .cra_driver_name = "stm32-ecb-aes",
1725		- .cra_priority = 200,
1726		- .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \|
1727		- CRYPTO_ALG_ASYNC,
1728		- .cra_blocksize = AES_BLOCK_SIZE,
1729		- .cra_ctxsize = sizeof(struct stm32_cryp_ctx),
1730		- .cra_alignmask = 0xf,
1731		- .cra_type = &crypto_ablkcipher_type,
1732		- .cra_module = THIS_MODULE,
1733		- .cra_init = stm32_cryp_cra_init,
1734		- .cra_ablkcipher = {
1735		- .min_keysize = AES_MIN_KEY_SIZE,
1736		- .max_keysize = AES_MAX_KEY_SIZE,
1737		- .setkey = stm32_cryp_aes_setkey,
1738		- .encrypt = stm32_cryp_aes_ecb_encrypt,
1739		- .decrypt = stm32_cryp_aes_ecb_decrypt,
1740		- }
	1545	+ .base.cra_name = "ecb(aes)",
	1546	+ .base.cra_driver_name = "stm32-ecb-aes",
	1547	+ .base.cra_priority = 200,
	1548	+ .base.cra_flags = CRYPTO_ALG_ASYNC,
	1549	+ .base.cra_blocksize = AES_BLOCK_SIZE,
	1550	+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
	1551	+ .base.cra_alignmask = 0,
	1552	+ .base.cra_module = THIS_MODULE,
	1553	+
	1554	+ .init = stm32_cryp_init_tfm,
	1555	+ .min_keysize = AES_MIN_KEY_SIZE,
	1556	+ .max_keysize = AES_MAX_KEY_SIZE,
	1557	+ .setkey = stm32_cryp_aes_setkey,
	1558	+ .encrypt = stm32_cryp_aes_ecb_encrypt,
	1559	+ .decrypt = stm32_cryp_aes_ecb_decrypt,
1741	1560	},
1742	1561	{
1743		- .cra_name = "cbc(aes)",
1744		- .cra_driver_name = "stm32-cbc-aes",
1745		- .cra_priority = 200,
1746		- .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \|
1747		- CRYPTO_ALG_ASYNC,
1748		- .cra_blocksize = AES_BLOCK_SIZE,
1749		- .cra_ctxsize = sizeof(struct stm32_cryp_ctx),
1750		- .cra_alignmask = 0xf,
1751		- .cra_type = &crypto_ablkcipher_type,
1752		- .cra_module = THIS_MODULE,
1753		- .cra_init = stm32_cryp_cra_init,
1754		- .cra_ablkcipher = {
1755		- .min_keysize = AES_MIN_KEY_SIZE,
1756		- .max_keysize = AES_MAX_KEY_SIZE,
1757		- .ivsize = AES_BLOCK_SIZE,
1758		- .setkey = stm32_cryp_aes_setkey,
1759		- .encrypt = stm32_cryp_aes_cbc_encrypt,
1760		- .decrypt = stm32_cryp_aes_cbc_decrypt,
1761		- }
	1562	+ .base.cra_name = "cbc(aes)",
	1563	+ .base.cra_driver_name = "stm32-cbc-aes",
	1564	+ .base.cra_priority = 200,
	1565	+ .base.cra_flags = CRYPTO_ALG_ASYNC,
	1566	+ .base.cra_blocksize = AES_BLOCK_SIZE,
	1567	+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
	1568	+ .base.cra_alignmask = 0,
	1569	+ .base.cra_module = THIS_MODULE,
	1570	+
	1571	+ .init = stm32_cryp_init_tfm,
	1572	+ .min_keysize = AES_MIN_KEY_SIZE,
	1573	+ .max_keysize = AES_MAX_KEY_SIZE,
	1574	+ .ivsize = AES_BLOCK_SIZE,
	1575	+ .setkey = stm32_cryp_aes_setkey,
	1576	+ .encrypt = stm32_cryp_aes_cbc_encrypt,
	1577	+ .decrypt = stm32_cryp_aes_cbc_decrypt,
1762	1578	},
1763	1579	{
1764		- .cra_name = "ctr(aes)",
1765		- .cra_driver_name = "stm32-ctr-aes",
1766		- .cra_priority = 200,
1767		- .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \|
1768		- CRYPTO_ALG_ASYNC,
1769		- .cra_blocksize = 1,
1770		- .cra_ctxsize = sizeof(struct stm32_cryp_ctx),
1771		- .cra_alignmask = 0xf,
1772		- .cra_type = &crypto_ablkcipher_type,
1773		- .cra_module = THIS_MODULE,
1774		- .cra_init = stm32_cryp_cra_init,
1775		- .cra_ablkcipher = {
1776		- .min_keysize = AES_MIN_KEY_SIZE,
1777		- .max_keysize = AES_MAX_KEY_SIZE,
1778		- .ivsize = AES_BLOCK_SIZE,
1779		- .setkey = stm32_cryp_aes_setkey,
1780		- .encrypt = stm32_cryp_aes_ctr_encrypt,
1781		- .decrypt = stm32_cryp_aes_ctr_decrypt,
1782		- }
	1580	+ .base.cra_name = "ctr(aes)",
	1581	+ .base.cra_driver_name = "stm32-ctr-aes",
	1582	+ .base.cra_priority = 200,
	1583	+ .base.cra_flags = CRYPTO_ALG_ASYNC,
	1584	+ .base.cra_blocksize = 1,
	1585	+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
	1586	+ .base.cra_alignmask = 0,
	1587	+ .base.cra_module = THIS_MODULE,
	1588	+
	1589	+ .init = stm32_cryp_init_tfm,
	1590	+ .min_keysize = AES_MIN_KEY_SIZE,
	1591	+ .max_keysize = AES_MAX_KEY_SIZE,
	1592	+ .ivsize = AES_BLOCK_SIZE,
	1593	+ .setkey = stm32_cryp_aes_setkey,
	1594	+ .encrypt = stm32_cryp_aes_ctr_encrypt,
	1595	+ .decrypt = stm32_cryp_aes_ctr_decrypt,
1783	1596	},
1784	1597	{
1785		- .cra_name = "ecb(des)",
1786		- .cra_driver_name = "stm32-ecb-des",
1787		- .cra_priority = 200,
1788		- .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \|
1789		- CRYPTO_ALG_ASYNC,
1790		- .cra_blocksize = DES_BLOCK_SIZE,
1791		- .cra_ctxsize = sizeof(struct stm32_cryp_ctx),
1792		- .cra_alignmask = 0xf,
1793		- .cra_type = &crypto_ablkcipher_type,
1794		- .cra_module = THIS_MODULE,
1795		- .cra_init = stm32_cryp_cra_init,
1796		- .cra_ablkcipher = {
1797		- .min_keysize = DES_BLOCK_SIZE,
1798		- .max_keysize = DES_BLOCK_SIZE,
1799		- .setkey = stm32_cryp_des_setkey,
1800		- .encrypt = stm32_cryp_des_ecb_encrypt,
1801		- .decrypt = stm32_cryp_des_ecb_decrypt,
1802		- }
	1598	+ .base.cra_name = "ecb(des)",
	1599	+ .base.cra_driver_name = "stm32-ecb-des",
	1600	+ .base.cra_priority = 200,
	1601	+ .base.cra_flags = CRYPTO_ALG_ASYNC,
	1602	+ .base.cra_blocksize = DES_BLOCK_SIZE,
	1603	+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
	1604	+ .base.cra_alignmask = 0,
	1605	+ .base.cra_module = THIS_MODULE,
	1606	+
	1607	+ .init = stm32_cryp_init_tfm,
	1608	+ .min_keysize = DES_BLOCK_SIZE,
	1609	+ .max_keysize = DES_BLOCK_SIZE,
	1610	+ .setkey = stm32_cryp_des_setkey,
	1611	+ .encrypt = stm32_cryp_des_ecb_encrypt,
	1612	+ .decrypt = stm32_cryp_des_ecb_decrypt,
1803	1613	},
1804	1614	{
1805		- .cra_name = "cbc(des)",
1806		- .cra_driver_name = "stm32-cbc-des",
1807		- .cra_priority = 200,
1808		- .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \|
1809		- CRYPTO_ALG_ASYNC,
1810		- .cra_blocksize = DES_BLOCK_SIZE,
1811		- .cra_ctxsize = sizeof(struct stm32_cryp_ctx),
1812		- .cra_alignmask = 0xf,
1813		- .cra_type = &crypto_ablkcipher_type,
1814		- .cra_module = THIS_MODULE,
1815		- .cra_init = stm32_cryp_cra_init,
1816		- .cra_ablkcipher = {
1817		- .min_keysize = DES_BLOCK_SIZE,
1818		- .max_keysize = DES_BLOCK_SIZE,
1819		- .ivsize = DES_BLOCK_SIZE,
1820		- .setkey = stm32_cryp_des_setkey,
1821		- .encrypt = stm32_cryp_des_cbc_encrypt,
1822		- .decrypt = stm32_cryp_des_cbc_decrypt,
1823		- }
	1615	+ .base.cra_name = "cbc(des)",
	1616	+ .base.cra_driver_name = "stm32-cbc-des",
	1617	+ .base.cra_priority = 200,
	1618	+ .base.cra_flags = CRYPTO_ALG_ASYNC,
	1619	+ .base.cra_blocksize = DES_BLOCK_SIZE,
	1620	+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
	1621	+ .base.cra_alignmask = 0,
	1622	+ .base.cra_module = THIS_MODULE,
	1623	+
	1624	+ .init = stm32_cryp_init_tfm,
	1625	+ .min_keysize = DES_BLOCK_SIZE,
	1626	+ .max_keysize = DES_BLOCK_SIZE,
	1627	+ .ivsize = DES_BLOCK_SIZE,
	1628	+ .setkey = stm32_cryp_des_setkey,
	1629	+ .encrypt = stm32_cryp_des_cbc_encrypt,
	1630	+ .decrypt = stm32_cryp_des_cbc_decrypt,
1824	1631	},
1825	1632	{
1826		- .cra_name = "ecb(des3_ede)",
1827		- .cra_driver_name = "stm32-ecb-des3",
1828		- .cra_priority = 200,
1829		- .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \|
1830		- CRYPTO_ALG_ASYNC,
1831		- .cra_blocksize = DES_BLOCK_SIZE,
1832		- .cra_ctxsize = sizeof(struct stm32_cryp_ctx),
1833		- .cra_alignmask = 0xf,
1834		- .cra_type = &crypto_ablkcipher_type,
1835		- .cra_module = THIS_MODULE,
1836		- .cra_init = stm32_cryp_cra_init,
1837		- .cra_ablkcipher = {
1838		- .min_keysize = 3 * DES_BLOCK_SIZE,
1839		- .max_keysize = 3 * DES_BLOCK_SIZE,
1840		- .setkey = stm32_cryp_tdes_setkey,
1841		- .encrypt = stm32_cryp_tdes_ecb_encrypt,
1842		- .decrypt = stm32_cryp_tdes_ecb_decrypt,
1843		- }
	1633	+ .base.cra_name = "ecb(des3_ede)",
	1634	+ .base.cra_driver_name = "stm32-ecb-des3",
	1635	+ .base.cra_priority = 200,
	1636	+ .base.cra_flags = CRYPTO_ALG_ASYNC,
	1637	+ .base.cra_blocksize = DES_BLOCK_SIZE,
	1638	+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
	1639	+ .base.cra_alignmask = 0,
	1640	+ .base.cra_module = THIS_MODULE,
	1641	+
	1642	+ .init = stm32_cryp_init_tfm,
	1643	+ .min_keysize = 3 * DES_BLOCK_SIZE,
	1644	+ .max_keysize = 3 * DES_BLOCK_SIZE,
	1645	+ .setkey = stm32_cryp_tdes_setkey,
	1646	+ .encrypt = stm32_cryp_tdes_ecb_encrypt,
	1647	+ .decrypt = stm32_cryp_tdes_ecb_decrypt,
1844	1648	},
1845	1649	{
1846		- .cra_name = "cbc(des3_ede)",
1847		- .cra_driver_name = "stm32-cbc-des3",
1848		- .cra_priority = 200,
1849		- .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER \|
1850		- CRYPTO_ALG_ASYNC,
1851		- .cra_blocksize = DES_BLOCK_SIZE,
1852		- .cra_ctxsize = sizeof(struct stm32_cryp_ctx),
1853		- .cra_alignmask = 0xf,
1854		- .cra_type = &crypto_ablkcipher_type,
1855		- .cra_module = THIS_MODULE,
1856		- .cra_init = stm32_cryp_cra_init,
1857		- .cra_ablkcipher = {
1858		- .min_keysize = 3 * DES_BLOCK_SIZE,
1859		- .max_keysize = 3 * DES_BLOCK_SIZE,
1860		- .ivsize = DES_BLOCK_SIZE,
1861		- .setkey = stm32_cryp_tdes_setkey,
1862		- .encrypt = stm32_cryp_tdes_cbc_encrypt,
1863		- .decrypt = stm32_cryp_tdes_cbc_decrypt,
1864		- }
	1650	+ .base.cra_name = "cbc(des3_ede)",
	1651	+ .base.cra_driver_name = "stm32-cbc-des3",
	1652	+ .base.cra_priority = 200,
	1653	+ .base.cra_flags = CRYPTO_ALG_ASYNC,
	1654	+ .base.cra_blocksize = DES_BLOCK_SIZE,
	1655	+ .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
	1656	+ .base.cra_alignmask = 0,
	1657	+ .base.cra_module = THIS_MODULE,
	1658	+
	1659	+ .init = stm32_cryp_init_tfm,
	1660	+ .min_keysize = 3 * DES_BLOCK_SIZE,
	1661	+ .max_keysize = 3 * DES_BLOCK_SIZE,
	1662	+ .ivsize = DES_BLOCK_SIZE,
	1663	+ .setkey = stm32_cryp_tdes_setkey,
	1664	+ .encrypt = stm32_cryp_tdes_cbc_encrypt,
	1665	+ .decrypt = stm32_cryp_tdes_cbc_decrypt,
1865	1666	},
1866	1667	};
1867	1668
..	..	@@ -1882,7 +1683,7 @@
1882	1683	.cra_flags = CRYPTO_ALG_ASYNC,
1883	1684	.cra_blocksize = 1,
1884	1685	.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
1885		- .cra_alignmask = 0xf,
	1686	+ .cra_alignmask = 0,
1886	1687	.cra_module = THIS_MODULE,
1887	1688	},
1888	1689	},
..	..	@@ -1902,7 +1703,7 @@
1902	1703	.cra_flags = CRYPTO_ALG_ASYNC,
1903	1704	.cra_blocksize = 1,
1904	1705	.cra_ctxsize = sizeof(struct stm32_cryp_ctx),
1905		- .cra_alignmask = 0xf,
	1706	+ .cra_alignmask = 0,
1906	1707	.cra_module = THIS_MODULE,
1907	1708	},
1908	1709	},
..	..	@@ -1929,7 +1730,6 @@
1929	1730	{
1930	1731	struct device *dev = &pdev->dev;
1931	1732	struct stm32_cryp *cryp;
1932		- struct resource *res;
1933	1733	struct reset_control *rst;
1934	1734	int irq, ret;
1935	1735
..	..	@@ -1943,18 +1743,13 @@
1943	1743
1944	1744	cryp->dev = dev;
1945	1745
1946		- mutex_init(&cryp->lock);
1947		-
1948		- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1949		- cryp->regs = devm_ioremap_resource(dev, res);
	1746	+ cryp->regs = devm_platform_ioremap_resource(pdev, 0);
1950	1747	if (IS_ERR(cryp->regs))
1951	1748	return PTR_ERR(cryp->regs);
1952	1749
1953	1750	irq = platform_get_irq(pdev, 0);
1954		- if (irq < 0) {
1955		- dev_err(dev, "Cannot get IRQ resource\n");
	1751	+ if (irq < 0)
1956	1752	return irq;
1957		- }
1958	1753
1959	1754	ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq,
1960	1755	stm32_cryp_irq_thread, IRQF_ONESHOT,
..	..	@@ -2010,7 +1805,7 @@
2010	1805	goto err_engine2;
2011	1806	}
2012	1807
2013		- ret = crypto_register_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
	1808	+ ret = crypto_register_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
2014	1809	if (ret) {
2015	1810	dev_err(dev, "Could not register algs\n");
2016	1811	goto err_algs;
..	..	@@ -2027,7 +1822,7 @@
2027	1822	return 0;
2028	1823
2029	1824	err_aead_algs:
2030		- crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
	1825	+ crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
2031	1826	err_algs:
2032	1827	err_engine2:
2033	1828	crypto_engine_exit(cryp->engine);
..	..	@@ -2052,12 +1847,12 @@
2052	1847	if (!cryp)
2053	1848	return -ENODEV;
2054	1849
2055		- ret = pm_runtime_get_sync(cryp->dev);
	1850	+ ret = pm_runtime_resume_and_get(cryp->dev);
2056	1851	if (ret < 0)
2057	1852	return ret;
2058	1853
2059	1854	crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));
2060		- crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
	1855	+ crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs));
2061	1856
2062	1857	crypto_engine_exit(cryp->engine);
2063	1858