~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Cryptographic API.
3	4	*
..	..	@@ -6,10 +7,6 @@
6	7	* Copyright (c) 2010 Nokia Corporation
7	8	* Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
8	9	* Copyright (c) 2011 Texas Instruments Incorporated
9		- *
10		- * This program is free software; you can redistribute it and/or modify
11		- * it under the terms of the GNU General Public License version 2 as published
12		- * by the Free Software Foundation.
13	10	*
14	11	* Some ideas are from old omap-sha1-md5.c driver.
15	12	*/
..	..	@@ -36,13 +33,13 @@
36	33	#include <linux/of_irq.h>
37	34	#include <linux/delay.h>
38	35	#include <linux/crypto.h>
39		-#include <linux/cryptohash.h>
40	36	#include <crypto/scatterwalk.h>
41	37	#include <crypto/algapi.h>
42	38	#include <crypto/sha.h>
43	39	#include <crypto/hash.h>
44	40	#include <crypto/hmac.h>
45	41	#include <crypto/internal/hash.h>
	42	+#include <crypto/engine.h>
46	43
47	44	#define MD5_DIGEST_SIZE 16
48	45
..	..	@@ -104,7 +101,6 @@
104	101	#define DEFAULT_AUTOSUSPEND_DELAY 1000
105	102
106	103	/* mostly device flags */
107		-#define FLAGS_BUSY 0
108	104	#define FLAGS_FINAL 1
109	105	#define FLAGS_DMA_ACTIVE 2
110	106	#define FLAGS_OUTPUT_READY 3
..	..	@@ -115,6 +111,8 @@
115	111	#define FLAGS_BE32_SHA1 8
116	112	#define FLAGS_SGS_COPIED 9
117	113	#define FLAGS_SGS_ALLOCED 10
	114	+#define FLAGS_HUGE 11
	115	+
118	116	/* context flags */
119	117	#define FLAGS_FINUP 16
120	118
..	..	@@ -139,12 +137,14 @@
139	137	#define BUFLEN SHA512_BLOCK_SIZE
140	138	#define OMAP_SHA_DMA_THRESHOLD 256
141	139
	140	+#define OMAP_SHA_MAX_DMA_LEN (1024 * 2048)
	141	+
142	142	struct omap_sham_dev;
143	143
144	144	struct omap_sham_reqctx {
145	145	struct omap_sham_dev *dd;
146	146	unsigned long flags;
147		- unsigned long op;
	147	+ u8 op;
148	148
149	149	u8 digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED;
150	150	size_t digcnt;
..	..	@@ -158,7 +158,7 @@
158	158	int sg_len;
159	159	unsigned int total; /* total request */
160	160
161		- u8 buffer[0] OMAP_ALIGNED;
	161	+ u8 buffer[] OMAP_ALIGNED;
162	162	};
163	163
164	164	struct omap_sham_hmac_ctx {
..	..	@@ -168,12 +168,13 @@
168	168	};
169	169
170	170	struct omap_sham_ctx {
	171	+ struct crypto_engine_ctx enginectx;
171	172	unsigned long flags;
172	173
173	174	/* fallback stuff */
174	175	struct crypto_shash *fallback;
175	176
176		- struct omap_sham_hmac_ctx base[0];
	177	+ struct omap_sham_hmac_ctx base[];
177	178	};
178	179
179	180	#define OMAP_SHAM_QUEUE_LENGTH 10
..	..	@@ -219,7 +220,6 @@
219	220	struct device *dev;
220	221	void __iomem *io_base;
221	222	int irq;
222		- spinlock_t lock;
223	223	int err;
224	224	struct dma_chan *dma_lch;
225	225	struct tasklet_struct done_task;
..	..	@@ -230,6 +230,7 @@
230	230	int fallback_sz;
231	231	struct crypto_queue queue;
232	232	struct ahash_request *req;
	233	+ struct crypto_engine *engine;
233	234
234	235	const struct omap_sham_pdata *pdata;
235	236	};
..	..	@@ -244,6 +245,9 @@
244	245	.dev_list = LIST_HEAD_INIT(sham.dev_list),
245	246	.lock = __SPIN_LOCK_UNLOCKED(sham.lock),
246	247	};
	248	+
	249	+static int omap_sham_enqueue(struct ahash_request *req, unsigned int op);
	250	+static void omap_sham_finish_req(struct ahash_request *req, int err);
247	251
248	252	static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset)
249	253	{
..	..	@@ -357,17 +361,17 @@
357	361
358	362	if (big_endian)
359	363	for (i = 0; i < d; i++)
360		- hash[i] = be32_to_cpu(in[i]);
	364	+ hash[i] = be32_to_cpup((__be32 *)in + i);
361	365	else
362	366	for (i = 0; i < d; i++)
363		- hash[i] = le32_to_cpu(in[i]);
	367	+ hash[i] = le32_to_cpup((__le32 *)in + i);
364	368	}
365	369
366	370	static int omap_sham_hw_init(struct omap_sham_dev *dd)
367	371	{
368	372	int err;
369	373
370		- err = pm_runtime_get_sync(dd->dev);
	374	+ err = pm_runtime_resume_and_get(dd->dev);
371	375	if (err < 0) {
372	376	dev_err(dd->dev, "failed to get sync: %d\n", err);
373	377	return err;
..	..	@@ -525,7 +529,7 @@
525	529	int mlen;
526	530	struct sg_mapping_iter mi;
527	531
528		- dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
	532	+ dev_dbg(dd->dev, "xmit_cpu: digcnt: %zd, length: %zd, final: %d\n",
529	533	ctx->digcnt, length, final);
530	534
531	535	dd->pdata->write_ctrl(dd, length, final, 0);
..	..	@@ -591,7 +595,7 @@
591	595	struct dma_slave_config cfg;
592	596	int ret;
593	597
594		- dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
	598	+ dev_dbg(dd->dev, "xmit_dma: digcnt: %zd, length: %zd, final: %d\n",
595	599	ctx->digcnt, length, final);
596	600
597	601	if (!dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE)) {
..	..	@@ -648,6 +652,8 @@
648	652	struct scatterlist *tmp;
649	653	int offset = ctx->offset;
650	654
	655	+ ctx->total = new_len;
	656	+
651	657	if (ctx->bufcnt)
652	658	n++;
653	659
..	..	@@ -665,15 +671,16 @@
665	671	sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt);
666	672	tmp = sg_next(tmp);
667	673	ctx->sg_len++;
	674	+ new_len -= ctx->bufcnt;
668	675	}
669	676
670	677	while (sg && new_len) {
671	678	int len = sg->length - offset;
672	679
673		- if (offset) {
	680	+ if (len <= 0) {
674	681	offset -= sg->length;
675		- if (offset < 0)
676		- offset = 0;
	682	+ sg = sg_next(sg);
	683	+ continue;
677	684	}
678	685
679	686	if (new_len < len)
..	..	@@ -681,33 +688,37 @@
681	688
682	689	if (len > 0) {
683	690	new_len -= len;
684		- sg_set_page(tmp, sg_page(sg), len, sg->offset);
685		- if (new_len <= 0)
686		- sg_mark_end(tmp);
687		- tmp = sg_next(tmp);
	691	+ sg_set_page(tmp, sg_page(sg), len, sg->offset + offset);
	692	+ offset = 0;
	693	+ ctx->offset = 0;
688	694	ctx->sg_len++;
	695	+ if (new_len <= 0)
	696	+ break;
	697	+ tmp = sg_next(tmp);
689	698	}
690	699
691	700	sg = sg_next(sg);
692	701	}
693	702
	703	+ if (tmp)
	704	+ sg_mark_end(tmp);
	705	+
694	706	set_bit(FLAGS_SGS_ALLOCED, &ctx->dd->flags);
695	707
	708	+ ctx->offset += new_len - ctx->bufcnt;
696	709	ctx->bufcnt = 0;
697	710
698	711	return 0;
699	712	}
700	713
701	714	static int omap_sham_copy_sgs(struct omap_sham_reqctx *ctx,
702		- struct scatterlist *sg, int bs, int new_len)
	715	+ struct scatterlist *sg, int bs,
	716	+ unsigned int new_len)
703	717	{
704	718	int pages;
705	719	void *buf;
706		- int len;
707	720
708		- len = new_len + ctx->bufcnt;
709		-
710		- pages = get_order(ctx->total);
	721	+ pages = get_order(new_len);
711	722
712	723	buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
713	724	if (!buf) {
..	..	@@ -719,14 +730,15 @@
719	730	memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt);
720	731
721	732	scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->offset,
722		- ctx->total - ctx->bufcnt, 0);
	733	+ min(new_len, ctx->total) - ctx->bufcnt, 0);
723	734	sg_init_table(ctx->sgl, 1);
724		- sg_set_buf(ctx->sgl, buf, len);
	735	+ sg_set_buf(ctx->sgl, buf, new_len);
725	736	ctx->sg = ctx->sgl;
726	737	set_bit(FLAGS_SGS_COPIED, &ctx->dd->flags);
727	738	ctx->sg_len = 1;
	739	+ ctx->offset += new_len - ctx->bufcnt;
728	740	ctx->bufcnt = 0;
729		- ctx->offset = 0;
	741	+ ctx->total = new_len;
730	742
731	743	return 0;
732	744	}
..	..	@@ -741,9 +753,19 @@
741	753	struct scatterlist *sg_tmp = sg;
742	754	int new_len;
743	755	int offset = rctx->offset;
	756	+ int bufcnt = rctx->bufcnt;
744	757
745		- if (!sg \|\| !sg->length \|\| !nbytes)
	758	+ if (!sg \|\| !sg->length \|\| !nbytes) {
	759	+ if (bufcnt) {
	760	+ bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs;
	761	+ sg_init_table(rctx->sgl, 1);
	762	+ sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, bufcnt);
	763	+ rctx->sg = rctx->sgl;
	764	+ rctx->sg_len = 1;
	765	+ }
	766	+
746	767	return 0;
	768	+ }
747	769
748	770	new_len = nbytes;
749	771
..	..	@@ -755,11 +777,27 @@
755	777	else
756	778	new_len = (new_len - 1) / bs * bs;
757	779
	780	+ if (!new_len)
	781	+ return 0;
	782	+
758	783	if (nbytes != new_len)
759	784	list_ok = false;
760	785
761	786	while (nbytes > 0 && sg_tmp) {
762	787	n++;
	788	+
	789	+ if (bufcnt) {
	790	+ if (!IS_ALIGNED(bufcnt, bs)) {
	791	+ aligned = false;
	792	+ break;
	793	+ }
	794	+ nbytes -= bufcnt;
	795	+ bufcnt = 0;
	796	+ if (!nbytes)
	797	+ list_ok = false;
	798	+
	799	+ continue;
	800	+ }
763	801
764	802	#ifdef CONFIG_ZONE_DMA
765	803	if (page_zonenum(sg_page(sg_tmp)) != ZONE_DMA) {
..	..	@@ -798,47 +836,68 @@
798	836	}
799	837	}
800	838
	839	+ if (new_len > OMAP_SHA_MAX_DMA_LEN) {
	840	+ new_len = OMAP_SHA_MAX_DMA_LEN;
	841	+ aligned = false;
	842	+ }
	843	+
801	844	if (!aligned)
802	845	return omap_sham_copy_sgs(rctx, sg, bs, new_len);
803	846	else if (!list_ok)
804	847	return omap_sham_copy_sg_lists(rctx, sg, bs, new_len);
805	848
	849	+ rctx->total = new_len;
	850	+ rctx->offset += new_len;
806	851	rctx->sg_len = n;
807		- rctx->sg = sg;
	852	+ if (rctx->bufcnt) {
	853	+ sg_init_table(rctx->sgl, 2);
	854	+ sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, rctx->bufcnt);
	855	+ sg_chain(rctx->sgl, 2, sg);
	856	+ rctx->sg = rctx->sgl;
	857	+ } else {
	858	+ rctx->sg = sg;
	859	+ }
808	860
809	861	return 0;
810	862	}
811	863
812		-static int omap_sham_prepare_request(struct ahash_request *req, bool update)
	864	+static int omap_sham_prepare_request(struct crypto_engine engine, void areq)
813	865	{
	866	+ struct ahash_request *req = container_of(areq, struct ahash_request,
	867	+ base);
814	868	struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
815	869	int bs;
816	870	int ret;
817		- int nbytes;
	871	+ unsigned int nbytes;
818	872	bool final = rctx->flags & BIT(FLAGS_FINUP);
819		- int xmit_len, hash_later;
	873	+ bool update = rctx->op == OP_UPDATE;
	874	+ int hash_later;
820	875
821	876	bs = get_block_size(rctx);
822	877
	878	+ nbytes = rctx->bufcnt;
	879	+
823	880	if (update)
824		- nbytes = req->nbytes;
825		- else
826		- nbytes = 0;
	881	+ nbytes += req->nbytes - rctx->offset;
827	882
828		- rctx->total = nbytes + rctx->bufcnt;
	883	+ dev_dbg(rctx->dd->dev,
	884	+ "%s: nbytes=%d, bs=%d, total=%d, offset=%d, bufcnt=%zd\n",
	885	+ __func__, nbytes, bs, rctx->total, rctx->offset,
	886	+ rctx->bufcnt);
829	887
830		- if (!rctx->total)
	888	+ if (!nbytes)
831	889	return 0;
832	890
833		- if (nbytes && (!IS_ALIGNED(rctx->bufcnt, bs))) {
	891	+ rctx->total = nbytes;
	892	+
	893	+ if (update && req->nbytes && (!IS_ALIGNED(rctx->bufcnt, bs))) {
834	894	int len = bs - rctx->bufcnt % bs;
835	895
836		- if (len > nbytes)
837		- len = nbytes;
	896	+ if (len > req->nbytes)
	897	+ len = req->nbytes;
838	898	scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, req->src,
839	899	0, len, 0);
840	900	rctx->bufcnt += len;
841		- nbytes -= len;
842	901	rctx->offset = len;
843	902	}
844	903
..	..	@@ -849,64 +908,25 @@
849	908	if (ret)
850	909	return ret;
851	910
852		- xmit_len = rctx->total;
853		-
854		- if (!IS_ALIGNED(xmit_len, bs)) {
855		- if (final)
856		- xmit_len = DIV_ROUND_UP(xmit_len, bs) * bs;
857		- else
858		- xmit_len = xmit_len / bs * bs;
859		- } else if (!final) {
860		- xmit_len -= bs;
861		- }
862		-
863		- hash_later = rctx->total - xmit_len;
	911	+ hash_later = nbytes - rctx->total;
864	912	if (hash_later < 0)
865	913	hash_later = 0;
866	914
867		- if (rctx->bufcnt && nbytes) {
868		- /* have data from previous operation and current */
869		- sg_init_table(rctx->sgl, 2);
870		- sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, rctx->bufcnt);
871		-
872		- sg_chain(rctx->sgl, 2, req->src);
873		-
874		- rctx->sg = rctx->sgl;
875		-
876		- rctx->sg_len++;
877		- } else if (rctx->bufcnt) {
878		- /* have buffered data only */
879		- sg_init_table(rctx->sgl, 1);
880		- sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, xmit_len);
881		-
882		- rctx->sg = rctx->sgl;
883		-
884		- rctx->sg_len = 1;
885		- }
886		-
887		- if (hash_later) {
888		- int offset = 0;
889		-
890		- if (hash_later > req->nbytes) {
891		- memcpy(rctx->buffer, rctx->buffer + xmit_len,
892		- hash_later - req->nbytes);
893		- offset = hash_later - req->nbytes;
894		- }
895		-
896		- if (req->nbytes) {
897		- scatterwalk_map_and_copy(rctx->buffer + offset,
898		- req->src,
899		- offset + req->nbytes -
900		- hash_later, hash_later, 0);
901		- }
	915	+ if (hash_later && hash_later <= rctx->buflen) {
	916	+ scatterwalk_map_and_copy(rctx->buffer,
	917	+ req->src,
	918	+ req->nbytes - hash_later,
	919	+ hash_later, 0);
902	920
903	921	rctx->bufcnt = hash_later;
904	922	} else {
905	923	rctx->bufcnt = 0;
906	924	}
907	925
908		- if (!final)
909		- rctx->total = xmit_len;
	926	+ if (hash_later > rctx->buflen)
	927	+ set_bit(FLAGS_HUGE, &rctx->dd->flags);
	928	+
	929	+ rctx->total = min(nbytes, rctx->total);
910	930
911	931	return 0;
912	932	}
..	..	@@ -922,7 +942,7 @@
922	942	return 0;
923	943	}
924	944
925		-struct omap_sham_dev omap_sham_find_dev(struct omap_sham_reqctx ctx)
	945	+static struct omap_sham_dev omap_sham_find_dev(struct omap_sham_reqctx ctx)
926	946	{
927	947	struct omap_sham_dev *dd;
928	948
..	..	@@ -1010,10 +1030,11 @@
1010	1030	struct ahash_request *req = dd->req;
1011	1031	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1012	1032	int err;
1013		- bool final = ctx->flags & BIT(FLAGS_FINUP);
	1033	+ bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
	1034	+ !(dd->flags & BIT(FLAGS_HUGE));
1014	1035
1015		- dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n",
1016		- ctx->total, ctx->digcnt, (ctx->flags & BIT(FLAGS_FINUP)) != 0);
	1036	+ dev_dbg(dd->dev, "update_req: total: %u, digcnt: %zd, final: %d",
	1037	+ ctx->total, ctx->digcnt, final);
1017	1038
1018	1039	if (ctx->total < get_block_size(ctx) \|\|
1019	1040	ctx->total < dd->fallback_sz)
..	..	@@ -1025,7 +1046,7 @@
1025	1046	err = omap_sham_xmit_dma(dd, ctx->total, final);
1026	1047
1027	1048	/* wait for dma completion before can take more data */
1028		- dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);
	1049	+ dev_dbg(dd->dev, "update: err: %d, digcnt: %zd\n", err, ctx->digcnt);
1029	1050
1030	1051	return err;
1031	1052	}
..	..	@@ -1035,6 +1056,9 @@
1035	1056	struct ahash_request *req = dd->req;
1036	1057	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1037	1058	int err = 0, use_dma = 1;
	1059	+
	1060	+ if (dd->flags & BIT(FLAGS_HUGE))
	1061	+ return 0;
1038	1062
1039	1063	if ((ctx->total <= get_block_size(ctx)) \|\| dd->polling_mode)
1040	1064	/*
..	..	@@ -1055,6 +1079,39 @@
1055	1079	return err;
1056	1080	}
1057	1081
	1082	+static int omap_sham_hash_one_req(struct crypto_engine engine, void areq)
	1083	+{
	1084	+ struct ahash_request *req = container_of(areq, struct ahash_request,
	1085	+ base);
	1086	+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
	1087	+ struct omap_sham_dev *dd = ctx->dd;
	1088	+ int err;
	1089	+ bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
	1090	+ !(dd->flags & BIT(FLAGS_HUGE));
	1091	+
	1092	+ dev_dbg(dd->dev, "hash-one: op: %u, total: %u, digcnt: %zd, final: %d",
	1093	+ ctx->op, ctx->total, ctx->digcnt, final);
	1094	+
	1095	+ dd->req = req;
	1096	+
	1097	+ err = omap_sham_hw_init(dd);
	1098	+ if (err)
	1099	+ return err;
	1100	+
	1101	+ if (ctx->digcnt)
	1102	+ dd->pdata->copy_hash(req, 0);
	1103	+
	1104	+ if (ctx->op == OP_UPDATE)
	1105	+ err = omap_sham_update_req(dd);
	1106	+ else if (ctx->op == OP_FINAL)
	1107	+ err = omap_sham_final_req(dd);
	1108	+
	1109	+ if (err != -EINPROGRESS)
	1110	+ omap_sham_finish_req(req, err);
	1111	+
	1112	+ return 0;
	1113	+}
	1114	+
1058	1115	static int omap_sham_finish_hmac(struct ahash_request *req)
1059	1116	{
1060	1117	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
..	..	@@ -1064,7 +1121,6 @@
1064	1121	SHASH_DESC_ON_STACK(shash, bctx->shash);
1065	1122
1066	1123	shash->tfm = bctx->shash;
1067		- shash->flags = 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */
1068	1124
1069	1125	return crypto_shash_init(shash) ?:
1070	1126	crypto_shash_update(shash, bctx->opad, bs) ?:
..	..	@@ -1084,7 +1140,7 @@
1084	1140	err = omap_sham_finish_hmac(req);
1085	1141	}
1086	1142
1087		- dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);
	1143	+ dev_dbg(dd->dev, "digcnt: %zd, bufcnt: %zd\n", ctx->digcnt, ctx->bufcnt);
1088	1144
1089	1145	return err;
1090	1146	}
..	..	@@ -1096,17 +1152,27 @@
1096	1152
1097	1153	if (test_bit(FLAGS_SGS_COPIED, &dd->flags))
1098	1154	free_pages((unsigned long)sg_virt(ctx->sg),
1099		- get_order(ctx->sg->length + ctx->bufcnt));
	1155	+ get_order(ctx->sg->length));
1100	1156
1101	1157	if (test_bit(FLAGS_SGS_ALLOCED, &dd->flags))
1102	1158	kfree(ctx->sg);
1103	1159
1104	1160	ctx->sg = NULL;
1105	1161
1106		- dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) \| BIT(FLAGS_SGS_COPIED));
	1162	+ dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) \| BIT(FLAGS_SGS_COPIED) \|
	1163	+ BIT(FLAGS_CPU) \| BIT(FLAGS_DMA_READY) \|
	1164	+ BIT(FLAGS_OUTPUT_READY));
	1165	+
	1166	+ if (!err)
	1167	+ dd->pdata->copy_hash(req, 1);
	1168	+
	1169	+ if (dd->flags & BIT(FLAGS_HUGE)) {
	1170	+ /* Re-enqueue the request */
	1171	+ omap_sham_enqueue(req, ctx->op);
	1172	+ return;
	1173	+ }
1107	1174
1108	1175	if (!err) {
1109		- dd->pdata->copy_hash(req, 1);
1110	1176	if (test_bit(FLAGS_FINAL, &dd->flags))
1111	1177	err = omap_sham_finish(req);
1112	1178	} else {
..	..	@@ -1114,87 +1180,21 @@
1114	1180	}
1115	1181
1116	1182	/* atomic operation is not needed here */
1117		- dd->flags &= ~(BIT(FLAGS_BUSY) \| BIT(FLAGS_FINAL) \| BIT(FLAGS_CPU) \|
	1183	+ dd->flags &= ~(BIT(FLAGS_FINAL) \| BIT(FLAGS_CPU) \|
1118	1184	BIT(FLAGS_DMA_READY) \| BIT(FLAGS_OUTPUT_READY));
1119	1185
1120	1186	pm_runtime_mark_last_busy(dd->dev);
1121	1187	pm_runtime_put_autosuspend(dd->dev);
1122	1188
1123		- if (req->base.complete)
1124		- req->base.complete(&req->base, err);
	1189	+ ctx->offset = 0;
	1190	+
	1191	+ crypto_finalize_hash_request(dd->engine, req, err);
1125	1192	}
1126	1193
1127	1194	static int omap_sham_handle_queue(struct omap_sham_dev *dd,
1128	1195	struct ahash_request *req)
1129	1196	{
1130		- struct crypto_async_request async_req, backlog;
1131		- struct omap_sham_reqctx *ctx;
1132		- unsigned long flags;
1133		- int err = 0, ret = 0;
1134		-
1135		-retry:
1136		- spin_lock_irqsave(&dd->lock, flags);
1137		- if (req)
1138		- ret = ahash_enqueue_request(&dd->queue, req);
1139		- if (test_bit(FLAGS_BUSY, &dd->flags)) {
1140		- spin_unlock_irqrestore(&dd->lock, flags);
1141		- return ret;
1142		- }
1143		- backlog = crypto_get_backlog(&dd->queue);
1144		- async_req = crypto_dequeue_request(&dd->queue);
1145		- if (async_req)
1146		- set_bit(FLAGS_BUSY, &dd->flags);
1147		- spin_unlock_irqrestore(&dd->lock, flags);
1148		-
1149		- if (!async_req)
1150		- return ret;
1151		-
1152		- if (backlog)
1153		- backlog->complete(backlog, -EINPROGRESS);
1154		-
1155		- req = ahash_request_cast(async_req);
1156		- dd->req = req;
1157		- ctx = ahash_request_ctx(req);
1158		-
1159		- err = omap_sham_prepare_request(req, ctx->op == OP_UPDATE);
1160		- if (err \|\| !ctx->total)
1161		- goto err1;
1162		-
1163		- dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
1164		- ctx->op, req->nbytes);
1165		-
1166		- err = omap_sham_hw_init(dd);
1167		- if (err)
1168		- goto err1;
1169		-
1170		- if (ctx->digcnt)
1171		- /* request has changed - restore hash */
1172		- dd->pdata->copy_hash(req, 0);
1173		-
1174		- if (ctx->op == OP_UPDATE) {
1175		- err = omap_sham_update_req(dd);
1176		- if (err != -EINPROGRESS && (ctx->flags & BIT(FLAGS_FINUP)))
1177		- /* no final() after finup() */
1178		- err = omap_sham_final_req(dd);
1179		- } else if (ctx->op == OP_FINAL) {
1180		- err = omap_sham_final_req(dd);
1181		- }
1182		-err1:
1183		- dev_dbg(dd->dev, "exit, err: %d\n", err);
1184		-
1185		- if (err != -EINPROGRESS) {
1186		- /* done_task will not finish it, so do it here */
1187		- omap_sham_finish_req(req, err);
1188		- req = NULL;
1189		-
1190		- /*
1191		- * Execute next request immediately if there is anything
1192		- * in queue.
1193		- */
1194		- goto retry;
1195		- }
1196		-
1197		- return ret;
	1197	+ return crypto_transfer_hash_request_to_engine(dd->engine, req);
1198	1198	}
1199	1199
1200	1200	static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
..	..	@@ -1228,17 +1228,6 @@
1228	1228	return omap_sham_enqueue(req, OP_UPDATE);
1229	1229	}
1230	1230
1231		-static int omap_sham_shash_digest(struct crypto_shash *tfm, u32 flags,
1232		- const u8 data, unsigned int len, u8 out)
1233		-{
1234		- SHASH_DESC_ON_STACK(shash, tfm);
1235		-
1236		- shash->tfm = tfm;
1237		- shash->flags = flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1238		-
1239		- return crypto_shash_digest(shash, data, len, out);
1240		-}
1241		-
1242	1231	static int omap_sham_final_shash(struct ahash_request *req)
1243	1232	{
1244	1233	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
..	..	@@ -1254,9 +1243,8 @@
1254	1243	!test_bit(FLAGS_AUTO_XOR, &ctx->dd->flags))
1255	1244	offset = get_block_size(ctx);
1256	1245
1257		- return omap_sham_shash_digest(tctx->fallback, req->base.flags,
1258		- ctx->buffer + offset,
1259		- ctx->bufcnt - offset, req->result);
	1246	+ return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer + offset,
	1247	+ ctx->bufcnt - offset, req->result);
1260	1248	}
1261	1249
1262	1250	static int omap_sham_final(struct ahash_request *req)
..	..	@@ -1322,9 +1310,8 @@
1322	1310	return err;
1323	1311
1324	1312	if (keylen > bs) {
1325		- err = omap_sham_shash_digest(bctx->shash,
1326		- crypto_shash_get_flags(bctx->shash),
1327		- key, keylen, bctx->ipad);
	1313	+ err = crypto_shash_tfm_digest(bctx->shash, key, keylen,
	1314	+ bctx->ipad);
1328	1315	if (err)
1329	1316	return err;
1330	1317	keylen = ds;
..	..	@@ -1376,6 +1363,10 @@
1376	1363	}
1377	1364
1378	1365	}
	1366	+
	1367	+ tctx->enginectx.op.do_one_request = omap_sham_hash_one_req;
	1368	+ tctx->enginectx.op.prepare_request = omap_sham_prepare_request;
	1369	+ tctx->enginectx.op.unprepare_request = NULL;
1379	1370
1380	1371	return 0;
1381	1372	}
..	..	@@ -1555,7 +1546,8 @@
1555	1546	.cra_name = "sha224",
1556	1547	.cra_driver_name = "omap-sha224",
1557	1548	.cra_priority = 400,
1558		- .cra_flags = CRYPTO_ALG_ASYNC \|
	1549	+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY \|
	1550	+ CRYPTO_ALG_ASYNC \|
1559	1551	CRYPTO_ALG_NEED_FALLBACK,
1560	1552	.cra_blocksize = SHA224_BLOCK_SIZE,
1561	1553	.cra_ctxsize = sizeof(struct omap_sham_ctx),
..	..	@@ -1576,7 +1568,8 @@
1576	1568	.cra_name = "sha256",
1577	1569	.cra_driver_name = "omap-sha256",
1578	1570	.cra_priority = 400,
1579		- .cra_flags = CRYPTO_ALG_ASYNC \|
	1571	+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY \|
	1572	+ CRYPTO_ALG_ASYNC \|
1580	1573	CRYPTO_ALG_NEED_FALLBACK,
1581	1574	.cra_blocksize = SHA256_BLOCK_SIZE,
1582	1575	.cra_ctxsize = sizeof(struct omap_sham_ctx),
..	..	@@ -1598,7 +1591,8 @@
1598	1591	.cra_name = "hmac(sha224)",
1599	1592	.cra_driver_name = "omap-hmac-sha224",
1600	1593	.cra_priority = 400,
1601		- .cra_flags = CRYPTO_ALG_ASYNC \|
	1594	+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY \|
	1595	+ CRYPTO_ALG_ASYNC \|
1602	1596	CRYPTO_ALG_NEED_FALLBACK,
1603	1597	.cra_blocksize = SHA224_BLOCK_SIZE,
1604	1598	.cra_ctxsize = sizeof(struct omap_sham_ctx) +
..	..	@@ -1621,7 +1615,8 @@
1621	1615	.cra_name = "hmac(sha256)",
1622	1616	.cra_driver_name = "omap-hmac-sha256",
1623	1617	.cra_priority = 400,
1624		- .cra_flags = CRYPTO_ALG_ASYNC \|
	1618	+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY \|
	1619	+ CRYPTO_ALG_ASYNC \|
1625	1620	CRYPTO_ALG_NEED_FALLBACK,
1626	1621	.cra_blocksize = SHA256_BLOCK_SIZE,
1627	1622	.cra_ctxsize = sizeof(struct omap_sham_ctx) +
..	..	@@ -1646,7 +1641,8 @@
1646	1641	.cra_name = "sha384",
1647	1642	.cra_driver_name = "omap-sha384",
1648	1643	.cra_priority = 400,
1649		- .cra_flags = CRYPTO_ALG_ASYNC \|
	1644	+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY \|
	1645	+ CRYPTO_ALG_ASYNC \|
1650	1646	CRYPTO_ALG_NEED_FALLBACK,
1651	1647	.cra_blocksize = SHA384_BLOCK_SIZE,
1652	1648	.cra_ctxsize = sizeof(struct omap_sham_ctx),
..	..	@@ -1667,7 +1663,8 @@
1667	1663	.cra_name = "sha512",
1668	1664	.cra_driver_name = "omap-sha512",
1669	1665	.cra_priority = 400,
1670		- .cra_flags = CRYPTO_ALG_ASYNC \|
	1666	+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY \|
	1667	+ CRYPTO_ALG_ASYNC \|
1671	1668	CRYPTO_ALG_NEED_FALLBACK,
1672	1669	.cra_blocksize = SHA512_BLOCK_SIZE,
1673	1670	.cra_ctxsize = sizeof(struct omap_sham_ctx),
..	..	@@ -1689,7 +1686,8 @@
1689	1686	.cra_name = "hmac(sha384)",
1690	1687	.cra_driver_name = "omap-hmac-sha384",
1691	1688	.cra_priority = 400,
1692		- .cra_flags = CRYPTO_ALG_ASYNC \|
	1689	+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY \|
	1690	+ CRYPTO_ALG_ASYNC \|
1693	1691	CRYPTO_ALG_NEED_FALLBACK,
1694	1692	.cra_blocksize = SHA384_BLOCK_SIZE,
1695	1693	.cra_ctxsize = sizeof(struct omap_sham_ctx) +
..	..	@@ -1712,7 +1710,8 @@
1712	1710	.cra_name = "hmac(sha512)",
1713	1711	.cra_driver_name = "omap-hmac-sha512",
1714	1712	.cra_priority = 400,
1715		- .cra_flags = CRYPTO_ALG_ASYNC \|
	1713	+ .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY \|
	1714	+ CRYPTO_ALG_ASYNC \|
1716	1715	CRYPTO_ALG_NEED_FALLBACK,
1717	1716	.cra_blocksize = SHA512_BLOCK_SIZE,
1718	1717	.cra_ctxsize = sizeof(struct omap_sham_ctx) +
..	..	@@ -1730,10 +1729,7 @@
1730	1729	struct omap_sham_dev dd = (struct omap_sham_dev )data;
1731	1730	int err = 0;
1732	1731
1733		- if (!test_bit(FLAGS_BUSY, &dd->flags)) {
1734		- omap_sham_handle_queue(dd, NULL);
1735		- return;
1736		- }
	1732	+ dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags);
1737	1733
1738	1734	if (test_bit(FLAGS_CPU, &dd->flags)) {
1739	1735	if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags))
..	..	@@ -1759,20 +1755,12 @@
1759	1755	dev_dbg(dd->dev, "update done: err: %d\n", err);
1760	1756	/* finish curent request */
1761	1757	omap_sham_finish_req(dd->req, err);
1762		-
1763		- /* If we are not busy, process next req */
1764		- if (!test_bit(FLAGS_BUSY, &dd->flags))
1765		- omap_sham_handle_queue(dd, NULL);
1766	1758	}
1767	1759
1768	1760	static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd)
1769	1761	{
1770		- if (!test_bit(FLAGS_BUSY, &dd->flags)) {
1771		- dev_warn(dd->dev, "Interrupt when no active requests.\n");
1772		- } else {
1773		- set_bit(FLAGS_OUTPUT_READY, &dd->flags);
1774		- tasklet_schedule(&dd->done_task);
1775		- }
	1762	+ set_bit(FLAGS_OUTPUT_READY, &dd->flags);
	1763	+ tasklet_schedule(&dd->done_task);
1776	1764
1777	1765	return IRQ_HANDLED;
1778	1766	}
..	..	@@ -1989,7 +1977,6 @@
1989	1977	/* Get the IRQ */
1990	1978	dd->irq = platform_get_irq(pdev, 0);
1991	1979	if (dd->irq < 0) {
1992		- dev_err(dev, "no IRQ resource info\n");
1993	1980	err = dd->irq;
1994	1981	goto err;
1995	1982	}
..	..	@@ -2046,7 +2033,6 @@
2046	2033	struct omap_sham_dev *dd = dev_get_drvdata(dev);
2047	2034	ssize_t status;
2048	2035	long value;
2049		- unsigned long flags;
2050	2036
2051	2037	status = kstrtol(buf, 0, &value);
2052	2038	if (status)
..	..	@@ -2060,9 +2046,7 @@
2060	2046	* than current size, it will just not accept new entries until
2061	2047	* it has shrank enough.
2062	2048	*/
2063		- spin_lock_irqsave(&dd->lock, flags);
2064	2049	dd->queue.max_qlen = value;
2065		- spin_unlock_irqrestore(&dd->lock, flags);
2066	2050
2067	2051	return size;
2068	2052	}
..	..	@@ -2099,7 +2083,6 @@
2099	2083	platform_set_drvdata(pdev, dd);
2100	2084
2101	2085	INIT_LIST_HEAD(&dd->list);
2102		- spin_lock_init(&dd->lock);
2103	2086	tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
2104	2087	crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
2105	2088
..	..	@@ -2160,9 +2143,19 @@
2160	2143	(rev & dd->pdata->major_mask) >> dd->pdata->major_shift,
2161	2144	(rev & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
2162	2145
2163		- spin_lock(&sham.lock);
	2146	+ spin_lock_bh(&sham.lock);
2164	2147	list_add_tail(&dd->list, &sham.dev_list);
2165		- spin_unlock(&sham.lock);
	2148	+ spin_unlock_bh(&sham.lock);
	2149	+
	2150	+ dd->engine = crypto_engine_alloc_init(dev, 1);
	2151	+ if (!dd->engine) {
	2152	+ err = -ENOMEM;
	2153	+ goto err_engine;
	2154	+ }
	2155	+
	2156	+ err = crypto_engine_start(dd->engine);
	2157	+ if (err)
	2158	+ goto err_engine_start;
2166	2159
2167	2160	for (i = 0; i < dd->pdata->algs_info_size; i++) {
2168	2161	if (dd->pdata->algs_info[i].registered)
..	..	@@ -2197,6 +2190,12 @@
2197	2190	for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
2198	2191	crypto_unregister_ahash(
2199	2192	&dd->pdata->algs_info[i].algs_list[j]);
	2193	+err_engine_start:
	2194	+ crypto_engine_exit(dd->engine);
	2195	+err_engine:
	2196	+ spin_lock_bh(&sham.lock);
	2197	+ list_del(&dd->list);
	2198	+ spin_unlock_bh(&sham.lock);
2200	2199	err_pm:
2201	2200	pm_runtime_disable(dev);
2202	2201	if (!dd->polling_mode)
..	..	@@ -2215,9 +2214,9 @@
2215	2214	dd = platform_get_drvdata(pdev);
2216	2215	if (!dd)
2217	2216	return -ENODEV;
2218		- spin_lock(&sham.lock);
	2217	+ spin_lock_bh(&sham.lock);
2219	2218	list_del(&dd->list);
2220		- spin_unlock(&sham.lock);
	2219	+ spin_unlock_bh(&sham.lock);
2221	2220	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
2222	2221	for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
2223	2222	crypto_unregister_ahash(
..	..	@@ -2229,6 +2228,8 @@
2229	2228
2230	2229	if (!dd->polling_mode)
2231	2230	dma_release_channel(dd->dma_lch);
	2231	+
	2232	+ sysfs_remove_group(&dd->dev->kobj, &omap_sham_attr_group);
2232	2233
2233	2234	return 0;
2234	2235	}
..	..	@@ -2242,7 +2243,7 @@
2242	2243
2243	2244	static int omap_sham_resume(struct device *dev)
2244	2245	{
2245		- int err = pm_runtime_get_sync(dev);
	2246	+ int err = pm_runtime_resume_and_get(dev);
2246	2247	if (err < 0) {
2247	2248	dev_err(dev, "failed to get sync: %d\n", err);
2248	2249	return err;