forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 37f49e37ab4cb5d0bc4c60eb5c6d4dd57db767bb 
 kernel/drivers/crypto/caam/caamalg_desc.c
....@@ -1,7 +1,8 @@
1
+// SPDX-License-Identifier: GPL-2.0+
12 /*
2
- * Shared descriptors for aead, ablkcipher algorithms
3
+ * Shared descriptors for aead, skcipher algorithms
34  *
4
- * Copyright 2016 NXP
5
+ * Copyright 2016-2019 NXP
56  */
67 
78 #include "compat.h"
....@@ -32,12 +33,11 @@
3233 	}
3334 
3435 	jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
35
-	append_operation(desc, type | OP_ALG_AS_INITFINAL |
36
-			 OP_ALG_DECRYPT);
36
+	append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
3737 	uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
3838 	set_jump_tgt_here(desc, jump_cmd);
39
-	append_operation(desc, type | OP_ALG_AS_INITFINAL |
40
-			 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
39
+	append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
40
+			 OP_ALG_AAI_DK);
4141 	set_jump_tgt_here(desc, uncond_jump_cmd);
4242 }
4343 
....@@ -114,11 +114,9 @@
114114 	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
115115 			 LDST_SRCDST_BYTE_CONTEXT);
116116 
117
-#ifdef DEBUG
118
-	print_hex_dump(KERN_ERR,
119
-		       "aead null enc shdesc@" __stringify(__LINE__)": ",
120
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
121
-#endif
117
+	print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
118
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
119
+			     1);
122120 }
123121 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
124122 
....@@ -203,11 +201,9 @@
203201 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
204202 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
205203 
206
-#ifdef DEBUG
207
-	print_hex_dump(KERN_ERR,
208
-		       "aead null dec shdesc@" __stringify(__LINE__)": ",
209
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
210
-#endif
204
+	print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
205
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
206
+			     1);
211207 }
212208 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
213209 
....@@ -357,10 +353,9 @@
357353 	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
358354 			 LDST_SRCDST_BYTE_CONTEXT);
359355 
360
-#ifdef DEBUG
361
-	print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
362
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
363
-#endif
356
+	print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
357
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
358
+			     1);
364359 }
365360 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
366361 
....@@ -378,6 +373,7 @@
378373  *         with OP_ALG_AAI_HMAC_PRECOMP.
379374  * @ivsize: initialization vector size
380375  * @icvsize: integrity check value (ICV) size (truncated or full)
376
+ * @geniv: whether to generate Encrypted Chain IV
381377  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
382378  * @nonce: pointer to rfc3686 nonce
383379  * @ctx1_iv_off: IV offset in CONTEXT1 register
....@@ -474,10 +470,9 @@
474470 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
475471 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
476472 
477
-#ifdef DEBUG
478
-	print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
479
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
480
-#endif
473
+	print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
474
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
475
+			     1);
481476 }
482477 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
483478 
....@@ -621,11 +616,9 @@
621616 	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
622617 			 LDST_SRCDST_BYTE_CONTEXT);
623618 
624
-#ifdef DEBUG
625
-	print_hex_dump(KERN_ERR,
626
-		       "aead givenc shdesc@" __stringify(__LINE__)": ",
627
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
628
-#endif
619
+	print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
620
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
621
+			     1);
629622 }
630623 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
631624 
....@@ -750,10 +743,9 @@
750743 	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
751744 			 LDST_SRCDST_BYTE_CONTEXT);
752745 
753
-#ifdef DEBUG
754
-	print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
755
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
756
-#endif
746
+	print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
747
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
748
+			     1);
757749 }
758750 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
759751 
....@@ -846,10 +838,9 @@
846838 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
847839 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
848840 
849
-#ifdef DEBUG
850
-	print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
851
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
852
-#endif
841
+	print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
842
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
843
+			     1);
853844 }
854845 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
855846 
....@@ -862,13 +853,16 @@
862853  * @ivsize: initialization vector size
863854  * @icvsize: integrity check value (ICV) size (truncated or full)
864855  * @is_qi: true when called from caam/qi
856
+ *
857
+ * Input sequence: AAD | PTXT
858
+ * Output sequence: AAD | CTXT | ICV
859
+ * AAD length (assoclen), which includes the IV length, is available in Math3.
865860  */
866861 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
867862 			       unsigned int ivsize, unsigned int icvsize,
868863 			       const bool is_qi)
869864 {
870
-	u32 *key_jump_cmd;
871
-
865
+	u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions;
872866 	init_sh_desc(desc, HDR_SHARE_SERIAL);
873867 
874868 	/* Skip key loading if it is loaded due to sharing */
....@@ -911,24 +905,26 @@
911905 	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
912906 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
913907 
914
-	/* Read assoc data */
908
+	/* Skip AAD */
909
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
910
+
911
+	/* Read cryptlen and set this value into VARSEQOUTLEN */
912
+	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
913
+
914
+	/* If cryptlen is ZERO jump to AAD command */
915
+	zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
916
+					    JUMP_COND_MATH_Z);
917
+
918
+	/* Read AAD data */
915919 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
916920 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
917921 
922
+	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
923
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA);
924
+
918925 	/* Skip IV */
919926 	append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
920
-
921
-	/* Will read cryptlen bytes */
922
-	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
923
-
924
-	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
925
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
926
-
927
-	/* Skip assoc data */
928
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
929
-
930
-	/* cryptlen = seqoutlen - assoclen */
931
-	append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
927
+	append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ);
932928 
933929 	/* Write encrypted data */
934930 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
....@@ -937,15 +933,25 @@
937933 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
938934 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
939935 
936
+	/* Jump instructions to avoid double reading of AAD */
937
+	skip_instructions = append_jump(desc, JUMP_TEST_ALL);
938
+
939
+	/* There is no input data, cryptlen = 0 */
940
+	set_jump_tgt_here(desc, zero_cryptlen_jump_cmd);
941
+
942
+	/* Read AAD */
943
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
944
+			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
945
+
946
+	set_jump_tgt_here(desc, skip_instructions);
947
+
940948 	/* Write ICV */
941949 	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
942950 			 LDST_SRCDST_BYTE_CONTEXT);
943951 
944
-#ifdef DEBUG
945
-	print_hex_dump(KERN_ERR,
946
-		       "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
947
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
948
-#endif
952
+	print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
953
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
954
+			     1);
949955 }
950956 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
951957 
....@@ -1038,11 +1044,9 @@
10381044 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
10391045 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
10401046 
1041
-#ifdef DEBUG
1042
-	print_hex_dump(KERN_ERR,
1043
-		       "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1044
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1045
-#endif
1047
+	print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1048
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1049
+			     1);
10461050 }
10471051 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
10481052 
....@@ -1123,11 +1127,9 @@
11231127 	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
11241128 			 LDST_SRCDST_BYTE_CONTEXT);
11251129 
1126
-#ifdef DEBUG
1127
-	print_hex_dump(KERN_ERR,
1128
-		       "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1129
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1130
-#endif
1130
+	print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1131
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1132
+			     1);
11311133 }
11321134 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
11331135 
....@@ -1213,19 +1215,147 @@
12131215 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
12141216 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
12151217 
1216
-#ifdef DEBUG
1217
-	print_hex_dump(KERN_ERR,
1218
-		       "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1219
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1220
-#endif
1218
+	print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1219
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1220
+			     1);
12211221 }
12221222 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
12231223 
1224
-/*
1225
- * For ablkcipher encrypt and decrypt, read from req->src and
1226
- * write to req->dst
1224
+/**
1225
+ * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1226
+ *                          IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1227
+ *                          descriptor (non-protocol).
1228
+ * @desc: pointer to buffer used for descriptor construction
1229
+ * @cdata: pointer to block cipher transform definitions
1230
+ *         Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1231
+ *         OP_ALG_AAI_AEAD.
1232
+ * @adata: pointer to authentication transform definitions
1233
+ *         Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1234
+ *         OP_ALG_AAI_AEAD.
1235
+ * @ivsize: initialization vector size
1236
+ * @icvsize: integrity check value (ICV) size (truncated or full)
1237
+ * @encap: true if encapsulation, false if decapsulation
1238
+ * @is_qi: true when called from caam/qi
12271239  */
1228
-static inline void ablkcipher_append_src_dst(u32 *desc)
1240
+void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1241
+			    struct alginfo *adata, unsigned int ivsize,
1242
+			    unsigned int icvsize, const bool encap,
1243
+			    const bool is_qi)
1244
+{
1245
+	u32 *key_jump_cmd, *wait_cmd;
1246
+	u32 nfifo;
1247
+	const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1248
+
1249
+	/* Note: Context registers are saved. */
1250
+	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1251
+
1252
+	/* skip key loading if they are loaded due to sharing */
1253
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1254
+				   JUMP_COND_SHRD);
1255
+
1256
+	append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1257
+			  CLASS_1 | KEY_DEST_CLASS_REG);
1258
+
1259
+	/* For IPsec load the salt from keymat in the context register */
1260
+	if (is_ipsec)
1261
+		append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1262
+				   LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1263
+				   4 << LDST_OFFSET_SHIFT);
1264
+
1265
+	set_jump_tgt_here(desc, key_jump_cmd);
1266
+
1267
+	/* Class 2 and 1 operations: Poly & ChaCha */
1268
+	if (encap) {
1269
+		append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1270
+				 OP_ALG_ENCRYPT);
1271
+		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1272
+				 OP_ALG_ENCRYPT);
1273
+	} else {
1274
+		append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1275
+				 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1276
+		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1277
+				 OP_ALG_DECRYPT);
1278
+	}
1279
+
1280
+	if (is_qi) {
1281
+		u32 *wait_load_cmd;
1282
+		u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1283
+
1284
+		/* REG3 = assoclen */
1285
+		append_seq_load(desc, 4, LDST_CLASS_DECO |
1286
+				LDST_SRCDST_WORD_DECO_MATH3 |
1287
+				4 << LDST_OFFSET_SHIFT);
1288
+
1289
+		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1290
+					    JUMP_COND_CALM | JUMP_COND_NCP |
1291
+					    JUMP_COND_NOP | JUMP_COND_NIP |
1292
+					    JUMP_COND_NIFP);
1293
+		set_jump_tgt_here(desc, wait_load_cmd);
1294
+
1295
+		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1296
+				LDST_SRCDST_BYTE_CONTEXT |
1297
+				ctx1_iv_off << LDST_OFFSET_SHIFT);
1298
+	}
1299
+
1300
+	/*
1301
+	 * MAGIC with NFIFO
1302
+	 * Read associated data from the input and send them to class1 and
1303
+	 * class2 alignment blocks. From class1 send data to output fifo and
1304
+	 * then write it to memory since we don't need to encrypt AD.
1305
+	 */
1306
+	nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1307
+		NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1308
+	append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1309
+			    LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1310
+
1311
+	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1312
+	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1313
+	append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1314
+			     FIFOLD_CLASS_CLASS1 | LDST_VLF);
1315
+	append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1316
+			MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1317
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1318
+
1319
+	/* IPsec - copy IV at the output */
1320
+	if (is_ipsec)
1321
+		append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1322
+				      0x2 << 25);
1323
+
1324
+	wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1325
+			       JUMP_COND_NOP | JUMP_TEST_ALL);
1326
+	set_jump_tgt_here(desc, wait_cmd);
1327
+
1328
+	if (encap) {
1329
+		/* Read and write cryptlen bytes */
1330
+		append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1331
+		append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1332
+				CAAM_CMD_SZ);
1333
+		aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1334
+
1335
+		/* Write ICV */
1336
+		append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1337
+				 LDST_SRCDST_BYTE_CONTEXT);
1338
+	} else {
1339
+		/* Read and write cryptlen bytes */
1340
+		append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1341
+				CAAM_CMD_SZ);
1342
+		append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1343
+				CAAM_CMD_SZ);
1344
+		aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1345
+
1346
+		/* Load ICV for verification */
1347
+		append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1348
+				     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1349
+	}
1350
+
1351
+	print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1352
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1353
+			     1);
1354
+}
1355
+EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1356
+
1357
+/* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
1358
+static inline void skcipher_append_src_dst(u32 *desc)
12291359 {
12301360 	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
12311361 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
....@@ -1235,20 +1365,24 @@
12351365 }
12361366 
12371367 /**
1238
- * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor
1368
+ * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
12391369  * @desc: pointer to buffer used for descriptor construction
12401370  * @cdata: pointer to block cipher transform definitions
12411371  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1242
- *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
1372
+ *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1373
+ *                                - OP_ALG_ALGSEL_CHACHA20
12431374  * @ivsize: initialization vector size
12441375  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
12451376  * @ctx1_iv_off: IV offset in CONTEXT1 register
12461377  */
1247
-void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
1248
-				  unsigned int ivsize, const bool is_rfc3686,
1249
-				  const u32 ctx1_iv_off)
1378
+void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1379
+				unsigned int ivsize, const bool is_rfc3686,
1380
+				const u32 ctx1_iv_off)
12501381 {
12511382 	u32 *key_jump_cmd;
1383
+	u32 options = cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT;
1384
+	bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1385
+			    OP_ALG_ALGSEL_CHACHA20);
12521386 
12531387 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
12541388 	/* Skip if already shared */
....@@ -1273,9 +1407,11 @@
12731407 
12741408 	set_jump_tgt_here(desc, key_jump_cmd);
12751409 
1276
-	/* Load iv */
1277
-	append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1278
-			LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1410
+	/* Load IV, if there is one */
1411
+	if (ivsize)
1412
+		append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1413
+				LDST_CLASS_1_CCB | (ctx1_iv_off <<
1414
+				LDST_OFFSET_SHIFT));
12791415 
12801416 	/* Load counter into CONTEXT1 reg */
12811417 	if (is_rfc3686)
....@@ -1285,35 +1421,43 @@
12851421 				      LDST_OFFSET_SHIFT));
12861422 
12871423 	/* Load operation */
1288
-	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1289
-			 OP_ALG_ENCRYPT);
1424
+	if (is_chacha20)
1425
+		options |= OP_ALG_AS_FINALIZE;
1426
+	append_operation(desc, options);
12901427 
12911428 	/* Perform operation */
1292
-	ablkcipher_append_src_dst(desc);
1429
+	skcipher_append_src_dst(desc);
12931430 
1294
-#ifdef DEBUG
1295
-	print_hex_dump(KERN_ERR,
1296
-		       "ablkcipher enc shdesc@" __stringify(__LINE__)": ",
1297
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1298
-#endif
1431
+	/* Store IV */
1432
+	if (!is_chacha20 && ivsize)
1433
+		append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1434
+				 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1435
+				 LDST_OFFSET_SHIFT));
1436
+
1437
+	print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
1438
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1439
+			     1);
12991440 }
1300
-EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap);
1441
+EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
13011442 
13021443 /**
1303
- * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor
1444
+ * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
13041445  * @desc: pointer to buffer used for descriptor construction
13051446  * @cdata: pointer to block cipher transform definitions
13061447  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1307
- *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
1448
+ *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1449
+ *                                - OP_ALG_ALGSEL_CHACHA20
13081450  * @ivsize: initialization vector size
13091451  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
13101452  * @ctx1_iv_off: IV offset in CONTEXT1 register
13111453  */
1312
-void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
1313
-				  unsigned int ivsize, const bool is_rfc3686,
1314
-				  const u32 ctx1_iv_off)
1454
+void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1455
+				unsigned int ivsize, const bool is_rfc3686,
1456
+				const u32 ctx1_iv_off)
13151457 {
13161458 	u32 *key_jump_cmd;
1459
+	bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1460
+			    OP_ALG_ALGSEL_CHACHA20);
13171461 
13181462 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
13191463 	/* Skip if already shared */
....@@ -1338,9 +1482,11 @@
13381482 
13391483 	set_jump_tgt_here(desc, key_jump_cmd);
13401484 
1341
-	/* load IV */
1342
-	append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1343
-			LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1485
+	/* Load IV, if there is one */
1486
+	if (ivsize)
1487
+		append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1488
+				LDST_CLASS_1_CCB | (ctx1_iv_off <<
1489
+				LDST_OFFSET_SHIFT));
13441490 
13451491 	/* Load counter into CONTEXT1 reg */
13461492 	if (is_rfc3686)
....@@ -1351,111 +1497,33 @@
13511497 
13521498 	/* Choose operation */
13531499 	if (ctx1_iv_off)
1354
-		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1500
+		append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
13551501 				 OP_ALG_DECRYPT);
13561502 	else
13571503 		append_dec_op1(desc, cdata->algtype);
13581504 
13591505 	/* Perform operation */
1360
-	ablkcipher_append_src_dst(desc);
1506
+	skcipher_append_src_dst(desc);
13611507 
1362
-#ifdef DEBUG
1363
-	print_hex_dump(KERN_ERR,
1364
-		       "ablkcipher dec shdesc@" __stringify(__LINE__)": ",
1365
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1366
-#endif
1508
+	/* Store IV */
1509
+	if (!is_chacha20 && ivsize)
1510
+		append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1511
+				 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1512
+				 LDST_OFFSET_SHIFT));
1513
+
1514
+	print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
1515
+			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1516
+			     1);
13671517 }
1368
-EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap);
1518
+EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
13691519 
13701520 /**
1371
- * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor
1372
- *                                   with HW-generated initialization vector.
1373
- * @desc: pointer to buffer used for descriptor construction
1374
- * @cdata: pointer to block cipher transform definitions
1375
- *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1376
- *         with OP_ALG_AAI_CBC.
1377
- * @ivsize: initialization vector size
1378
- * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1379
- * @ctx1_iv_off: IV offset in CONTEXT1 register
1380
- */
1381
-void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata,
1382
-				     unsigned int ivsize, const bool is_rfc3686,
1383
-				     const u32 ctx1_iv_off)
1384
-{
1385
-	u32 *key_jump_cmd, geniv;
1386
-
1387
-	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1388
-	/* Skip if already shared */
1389
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1390
-				   JUMP_COND_SHRD);
1391
-
1392
-	/* Load class1 key only */
1393
-	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1394
-			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1395
-
1396
-	/* Load Nonce into CONTEXT1 reg */
1397
-	if (is_rfc3686) {
1398
-		const u8 *nonce = cdata->key_virt + cdata->keylen;
1399
-
1400
-		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1401
-				   LDST_CLASS_IND_CCB |
1402
-				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1403
-		append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1404
-			    MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1405
-			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1406
-	}
1407
-	set_jump_tgt_here(desc, key_jump_cmd);
1408
-
1409
-	/* Generate IV */
1410
-	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1411
-		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND |
1412
-		(ivsize << NFIFOENTRY_DLEN_SHIFT);
1413
-	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1414
-			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1415
-	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1416
-	append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO |
1417
-		    MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) |
1418
-		    (ctx1_iv_off << MOVE_OFFSET_SHIFT));
1419
-	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1420
-
1421
-	/* Copy generated IV to memory */
1422
-	append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1423
-			 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1424
-
1425
-	/* Load Counter into CONTEXT1 reg */
1426
-	if (is_rfc3686)
1427
-		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1428
-				     LDST_SRCDST_BYTE_CONTEXT |
1429
-				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1430
-				      LDST_OFFSET_SHIFT));
1431
-
1432
-	if (ctx1_iv_off)
1433
-		append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
1434
-			    (1 << JUMP_OFFSET_SHIFT));
1435
-
1436
-	/* Load operation */
1437
-	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1438
-			 OP_ALG_ENCRYPT);
1439
-
1440
-	/* Perform operation */
1441
-	ablkcipher_append_src_dst(desc);
1442
-
1443
-#ifdef DEBUG
1444
-	print_hex_dump(KERN_ERR,
1445
-		       "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
1446
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1447
-#endif
1448
-}
1449
-EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap);
1450
-
1451
-/**
1452
- * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared
1453
- *                                    descriptor
1521
+ * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
14541522  * @desc: pointer to buffer used for descriptor construction
14551523  * @cdata: pointer to block cipher transform definitions
14561524  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
14571525  */
1458
-void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata)
1526
+void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
14591527 {
14601528 	/*
14611529 	 * Set sector size to a big value, practically disabling
....@@ -1483,37 +1551,41 @@
14831551 	set_jump_tgt_here(desc, key_jump_cmd);
14841552 
14851553 	/*
1486
-	 * create sequence for loading the sector index
1487
-	 * Upper 8B of IV - will be used as sector index
1488
-	 * Lower 8B of IV - will be discarded
1554
+	 * create sequence for loading the sector index / 16B tweak value
1555
+	 * Lower 8B of IV - sector index / tweak lower half
1556
+	 * Upper 8B of IV - upper half of 16B tweak
14891557 	 */
14901558 	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
14911559 			(0x20 << LDST_OFFSET_SHIFT));
1492
-	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1560
+	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1561
+			(0x30 << LDST_OFFSET_SHIFT));
14931562 
14941563 	/* Load operation */
14951564 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
14961565 			 OP_ALG_ENCRYPT);
14971566 
14981567 	/* Perform operation */
1499
-	ablkcipher_append_src_dst(desc);
1568
+	skcipher_append_src_dst(desc);
15001569 
1501
-#ifdef DEBUG
1502
-	print_hex_dump(KERN_ERR,
1503
-		       "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
1504
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1505
-#endif
1570
+	/* Store lower 8B and upper 8B of IV */
1571
+	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1572
+			 (0x20 << LDST_OFFSET_SHIFT));
1573
+	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1574
+			 (0x30 << LDST_OFFSET_SHIFT));
1575
+
1576
+	print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
1577
+			     ": ", DUMP_PREFIX_ADDRESS, 16, 4,
1578
+			     desc, desc_bytes(desc), 1);
15061579 }
1507
-EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap);
1580
+EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
15081581 
15091582 /**
1510
- * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared
1511
- *                                    descriptor
1583
+ * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
15121584  * @desc: pointer to buffer used for descriptor construction
15131585  * @cdata: pointer to block cipher transform definitions
15141586  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
15151587  */
1516
-void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata)
1588
+void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
15171589 {
15181590 	/*
15191591 	 * Set sector size to a big value, practically disabling
....@@ -1541,27 +1613,31 @@
15411613 	set_jump_tgt_here(desc, key_jump_cmd);
15421614 
15431615 	/*
1544
-	 * create sequence for loading the sector index
1545
-	 * Upper 8B of IV - will be used as sector index
1546
-	 * Lower 8B of IV - will be discarded
1616
+	 * create sequence for loading the sector index / 16B tweak value
1617
+	 * Lower 8B of IV - sector index / tweak lower half
1618
+	 * Upper 8B of IV - upper half of 16B tweak
15471619 	 */
15481620 	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
15491621 			(0x20 << LDST_OFFSET_SHIFT));
1550
-	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1551
-
1622
+	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1623
+			(0x30 << LDST_OFFSET_SHIFT));
15521624 	/* Load operation */
15531625 	append_dec_op1(desc, cdata->algtype);
15541626 
15551627 	/* Perform operation */
1556
-	ablkcipher_append_src_dst(desc);
1628
+	skcipher_append_src_dst(desc);
15571629 
1558
-#ifdef DEBUG
1559
-	print_hex_dump(KERN_ERR,
1560
-		       "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
1561
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1562
-#endif
1630
+	/* Store lower 8B and upper 8B of IV */
1631
+	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1632
+			 (0x20 << LDST_OFFSET_SHIFT));
1633
+	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1634
+			 (0x30 << LDST_OFFSET_SHIFT));
1635
+
1636
+	print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
1637
+			     ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
1638
+			     desc_bytes(desc), 1);
15631639 }
1564
-EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap);
1640
+EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
15651641 
15661642 MODULE_LICENSE("GPL");
15671643 MODULE_DESCRIPTION("FSL CAAM descriptor support");