.. | .. |
---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ |
---|
1 | 2 | /* |
---|
2 | 3 | * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with |
---|
3 | 4 | * Crypto Extensions |
---|
4 | 5 | * |
---|
5 | 6 | * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> |
---|
6 | | - * |
---|
7 | | - * This program is free software; you can redistribute it and/or modify |
---|
8 | | - * it under the terms of the GNU General Public License version 2 as |
---|
9 | | - * published by the Free Software Foundation. |
---|
10 | 7 | */ |
---|
11 | 8 | |
---|
12 | 9 | #include <linux/linkage.h> |
---|
13 | 10 | #include <asm/assembler.h> |
---|
14 | 11 | |
---|
15 | | -#define AES_ENTRY(func) ENTRY(ce_ ## func) |
---|
16 | | -#define AES_ENDPROC(func) ENDPROC(ce_ ## func) |
---|
| 12 | +#define AES_FUNC_START(func) SYM_FUNC_START(ce_ ## func) |
---|
| 13 | +#define AES_FUNC_END(func) SYM_FUNC_END(ce_ ## func) |
---|
17 | 14 | |
---|
18 | 15 | .arch armv8-a+crypto |
---|
| 16 | + |
---|
| 17 | + xtsmask .req v16 |
---|
| 18 | + cbciv .req v16 |
---|
| 19 | + vctr .req v16 |
---|
| 20 | + |
---|
| 21 | + .macro xts_reload_mask, tmp |
---|
| 22 | + .endm |
---|
| 23 | + |
---|
| 24 | + .macro xts_cts_skip_tw, reg, lbl |
---|
| 25 | + .endm |
---|
19 | 26 | |
---|
20 | 27 | /* preload all round keys */ |
---|
21 | 28 | .macro load_round_keys, rounds, rk |
---|
.. | .. |
---|
47 | 54 | load_round_keys \rounds, \temp |
---|
48 | 55 | .endm |
---|
49 | 56 | |
---|
50 | | - .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3 |
---|
| 57 | + .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4 |
---|
51 | 58 | aes\de \i0\().16b, \k\().16b |
---|
52 | 59 | aes\mc \i0\().16b, \i0\().16b |
---|
53 | 60 | .ifnb \i1 |
---|
.. | .. |
---|
58 | 65 | aes\mc \i2\().16b, \i2\().16b |
---|
59 | 66 | aes\de \i3\().16b, \k\().16b |
---|
60 | 67 | aes\mc \i3\().16b, \i3\().16b |
---|
| 68 | + .ifnb \i4 |
---|
| 69 | + aes\de \i4\().16b, \k\().16b |
---|
| 70 | + aes\mc \i4\().16b, \i4\().16b |
---|
| 71 | + .endif |
---|
61 | 72 | .endif |
---|
62 | 73 | .endif |
---|
63 | 74 | .endm |
---|
64 | 75 | |
---|
65 | | - /* up to 4 interleaved encryption rounds with the same round key */ |
---|
66 | | - .macro round_Nx, enc, k, i0, i1, i2, i3 |
---|
| 76 | + /* up to 5 interleaved encryption rounds with the same round key */ |
---|
| 77 | + .macro round_Nx, enc, k, i0, i1, i2, i3, i4 |
---|
67 | 78 | .ifc \enc, e |
---|
68 | | - do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3 |
---|
| 79 | + do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3, \i4 |
---|
69 | 80 | .else |
---|
70 | | - do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3 |
---|
| 81 | + do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3, \i4 |
---|
71 | 82 | .endif |
---|
72 | 83 | .endm |
---|
73 | 84 | |
---|
74 | | - /* up to 4 interleaved final rounds */ |
---|
75 | | - .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3 |
---|
| 85 | + /* up to 5 interleaved final rounds */ |
---|
| 86 | + .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4 |
---|
76 | 87 | aes\de \i0\().16b, \k\().16b |
---|
77 | 88 | .ifnb \i1 |
---|
78 | 89 | aes\de \i1\().16b, \k\().16b |
---|
79 | 90 | .ifnb \i3 |
---|
80 | 91 | aes\de \i2\().16b, \k\().16b |
---|
81 | 92 | aes\de \i3\().16b, \k\().16b |
---|
| 93 | + .ifnb \i4 |
---|
| 94 | + aes\de \i4\().16b, \k\().16b |
---|
| 95 | + .endif |
---|
82 | 96 | .endif |
---|
83 | 97 | .endif |
---|
84 | 98 | eor \i0\().16b, \i0\().16b, \k2\().16b |
---|
.. | .. |
---|
87 | 101 | .ifnb \i3 |
---|
88 | 102 | eor \i2\().16b, \i2\().16b, \k2\().16b |
---|
89 | 103 | eor \i3\().16b, \i3\().16b, \k2\().16b |
---|
| 104 | + .ifnb \i4 |
---|
| 105 | + eor \i4\().16b, \i4\().16b, \k2\().16b |
---|
| 106 | + .endif |
---|
90 | 107 | .endif |
---|
91 | 108 | .endif |
---|
92 | 109 | .endm |
---|
93 | 110 | |
---|
94 | | - /* up to 4 interleaved blocks */ |
---|
95 | | - .macro do_block_Nx, enc, rounds, i0, i1, i2, i3 |
---|
| 111 | + /* up to 5 interleaved blocks */ |
---|
| 112 | + .macro do_block_Nx, enc, rounds, i0, i1, i2, i3, i4 |
---|
96 | 113 | cmp \rounds, #12 |
---|
97 | 114 | blo 2222f /* 128 bits */ |
---|
98 | 115 | beq 1111f /* 192 bits */ |
---|
99 | | - round_Nx \enc, v17, \i0, \i1, \i2, \i3 |
---|
100 | | - round_Nx \enc, v18, \i0, \i1, \i2, \i3 |
---|
101 | | -1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3 |
---|
102 | | - round_Nx \enc, v20, \i0, \i1, \i2, \i3 |
---|
| 116 | + round_Nx \enc, v17, \i0, \i1, \i2, \i3, \i4 |
---|
| 117 | + round_Nx \enc, v18, \i0, \i1, \i2, \i3, \i4 |
---|
| 118 | +1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3, \i4 |
---|
| 119 | + round_Nx \enc, v20, \i0, \i1, \i2, \i3, \i4 |
---|
103 | 120 | 2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29 |
---|
104 | | - round_Nx \enc, \key, \i0, \i1, \i2, \i3 |
---|
| 121 | + round_Nx \enc, \key, \i0, \i1, \i2, \i3, \i4 |
---|
105 | 122 | .endr |
---|
106 | | - fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3 |
---|
| 123 | + fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3, \i4 |
---|
107 | 124 | .endm |
---|
108 | 125 | |
---|
109 | 126 | .macro encrypt_block, in, rounds, t0, t1, t2 |
---|
110 | 127 | do_block_Nx e, \rounds, \in |
---|
111 | 128 | .endm |
---|
112 | 129 | |
---|
113 | | - .macro encrypt_block2x, i0, i1, rounds, t0, t1, t2 |
---|
114 | | - do_block_Nx e, \rounds, \i0, \i1 |
---|
115 | | - .endm |
---|
116 | | - |
---|
117 | 130 | .macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 |
---|
118 | 131 | do_block_Nx e, \rounds, \i0, \i1, \i2, \i3 |
---|
| 132 | + .endm |
---|
| 133 | + |
---|
| 134 | + .macro encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 |
---|
| 135 | + do_block_Nx e, \rounds, \i0, \i1, \i2, \i3, \i4 |
---|
119 | 136 | .endm |
---|
120 | 137 | |
---|
121 | 138 | .macro decrypt_block, in, rounds, t0, t1, t2 |
---|
122 | 139 | do_block_Nx d, \rounds, \in |
---|
123 | 140 | .endm |
---|
124 | 141 | |
---|
125 | | - .macro decrypt_block2x, i0, i1, rounds, t0, t1, t2 |
---|
126 | | - do_block_Nx d, \rounds, \i0, \i1 |
---|
127 | | - .endm |
---|
128 | | - |
---|
129 | 142 | .macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 |
---|
130 | 143 | do_block_Nx d, \rounds, \i0, \i1, \i2, \i3 |
---|
131 | 144 | .endm |
---|
132 | 145 | |
---|
| 146 | + .macro decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 |
---|
| 147 | + do_block_Nx d, \rounds, \i0, \i1, \i2, \i3, \i4 |
---|
| 148 | + .endm |
---|
| 149 | + |
---|
| 150 | +#define MAX_STRIDE 5 |
---|
| 151 | + |
---|
133 | 152 | #include "aes-modes.S" |
---|