| .. | .. |
|---|
| 14 | 14 | struct safexcel_desc_ring *cdr, |
|---|
| 15 | 15 | struct safexcel_desc_ring *rdr) |
|---|
| 16 | 16 | { |
|---|
| 17 | | - cdr->offset = sizeof(u32) * priv->config.cd_offset; |
|---|
| 17 | + int i; |
|---|
| 18 | + struct safexcel_command_desc *cdesc; |
|---|
| 19 | + dma_addr_t atok; |
|---|
| 20 | + |
|---|
| 21 | + /* Actual command descriptor ring */ |
|---|
| 22 | + cdr->offset = priv->config.cd_offset; |
|---|
| 18 | 23 | cdr->base = dmam_alloc_coherent(priv->dev, |
|---|
| 19 | 24 | cdr->offset * EIP197_DEFAULT_RING_SIZE, |
|---|
| 20 | 25 | &cdr->base_dma, GFP_KERNEL); |
|---|
| .. | .. |
|---|
| 24 | 29 | cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1); |
|---|
| 25 | 30 | cdr->read = cdr->base; |
|---|
| 26 | 31 | |
|---|
| 27 | | - rdr->offset = sizeof(u32) * priv->config.rd_offset; |
|---|
| 32 | + /* Command descriptor shadow ring for storing additional token data */ |
|---|
| 33 | + cdr->shoffset = priv->config.cdsh_offset; |
|---|
| 34 | + cdr->shbase = dmam_alloc_coherent(priv->dev, |
|---|
| 35 | + cdr->shoffset * |
|---|
| 36 | + EIP197_DEFAULT_RING_SIZE, |
|---|
| 37 | + &cdr->shbase_dma, GFP_KERNEL); |
|---|
| 38 | + if (!cdr->shbase) |
|---|
| 39 | + return -ENOMEM; |
|---|
| 40 | + cdr->shwrite = cdr->shbase; |
|---|
| 41 | + cdr->shbase_end = cdr->shbase + cdr->shoffset * |
|---|
| 42 | + (EIP197_DEFAULT_RING_SIZE - 1); |
|---|
| 43 | + |
|---|
| 44 | + /* |
|---|
| 45 | + * Populate command descriptors with physical pointers to shadow descs. |
|---|
| 46 | + * Note that we only need to do this once if we don't overwrite them. |
|---|
| 47 | + */ |
|---|
| 48 | + cdesc = cdr->base; |
|---|
| 49 | + atok = cdr->shbase_dma; |
|---|
| 50 | + for (i = 0; i < EIP197_DEFAULT_RING_SIZE; i++) { |
|---|
| 51 | + cdesc->atok_lo = lower_32_bits(atok); |
|---|
| 52 | + cdesc->atok_hi = upper_32_bits(atok); |
|---|
| 53 | + cdesc = (void *)cdesc + cdr->offset; |
|---|
| 54 | + atok += cdr->shoffset; |
|---|
| 55 | + } |
|---|
| 56 | + |
|---|
| 57 | + rdr->offset = priv->config.rd_offset; |
|---|
| 58 | + /* Use shoffset for result token offset here */ |
|---|
| 59 | + rdr->shoffset = priv->config.res_offset; |
|---|
| 28 | 60 | rdr->base = dmam_alloc_coherent(priv->dev, |
|---|
| 29 | 61 | rdr->offset * EIP197_DEFAULT_RING_SIZE, |
|---|
| 30 | 62 | &rdr->base_dma, GFP_KERNEL); |
|---|
| .. | .. |
|---|
| 42 | 74 | return (atomic_inc_return(&priv->ring_used) % priv->config.rings); |
|---|
| 43 | 75 | } |
|---|
| 44 | 76 | |
|---|
| 45 | | -static void *safexcel_ring_next_wptr(struct safexcel_crypto_priv *priv, |
|---|
| 46 | | - struct safexcel_desc_ring *ring) |
|---|
| 77 | +static void *safexcel_ring_next_cwptr(struct safexcel_crypto_priv *priv, |
|---|
| 78 | + struct safexcel_desc_ring *ring, |
|---|
| 79 | + bool first, |
|---|
| 80 | + struct safexcel_token **atoken) |
|---|
| 47 | 81 | { |
|---|
| 48 | 82 | void *ptr = ring->write; |
|---|
| 83 | + |
|---|
| 84 | + if (first) |
|---|
| 85 | + *atoken = ring->shwrite; |
|---|
| 86 | + |
|---|
| 87 | + if ((ring->write == ring->read - ring->offset) || |
|---|
| 88 | + (ring->read == ring->base && ring->write == ring->base_end)) |
|---|
| 89 | + return ERR_PTR(-ENOMEM); |
|---|
| 90 | + |
|---|
| 91 | + if (ring->write == ring->base_end) { |
|---|
| 92 | + ring->write = ring->base; |
|---|
| 93 | + ring->shwrite = ring->shbase; |
|---|
| 94 | + } else { |
|---|
| 95 | + ring->write += ring->offset; |
|---|
| 96 | + ring->shwrite += ring->shoffset; |
|---|
| 97 | + } |
|---|
| 98 | + |
|---|
| 99 | + return ptr; |
|---|
| 100 | +} |
|---|
| 101 | + |
|---|
| 102 | +static void *safexcel_ring_next_rwptr(struct safexcel_crypto_priv *priv, |
|---|
| 103 | + struct safexcel_desc_ring *ring, |
|---|
| 104 | + struct result_data_desc **rtoken) |
|---|
| 105 | +{ |
|---|
| 106 | + void *ptr = ring->write; |
|---|
| 107 | + |
|---|
| 108 | + /* Result token at relative offset shoffset */ |
|---|
| 109 | + *rtoken = ring->write + ring->shoffset; |
|---|
| 49 | 110 | |
|---|
| 50 | 111 | if ((ring->write == ring->read - ring->offset) || |
|---|
| 51 | 112 | (ring->read == ring->base && ring->write == ring->base_end)) |
|---|
| .. | .. |
|---|
| 106 | 167 | if (ring->write == ring->read) |
|---|
| 107 | 168 | return; |
|---|
| 108 | 169 | |
|---|
| 109 | | - if (ring->write == ring->base) |
|---|
| 170 | + if (ring->write == ring->base) { |
|---|
| 110 | 171 | ring->write = ring->base_end; |
|---|
| 111 | | - else |
|---|
| 172 | + ring->shwrite = ring->shbase_end; |
|---|
| 173 | + } else { |
|---|
| 112 | 174 | ring->write -= ring->offset; |
|---|
| 175 | + ring->shwrite -= ring->shoffset; |
|---|
| 176 | + } |
|---|
| 113 | 177 | } |
|---|
| 114 | 178 | |
|---|
| 115 | 179 | struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv, |
|---|
| .. | .. |
|---|
| 117 | 181 | bool first, bool last, |
|---|
| 118 | 182 | dma_addr_t data, u32 data_len, |
|---|
| 119 | 183 | u32 full_data_len, |
|---|
| 120 | | - dma_addr_t context) { |
|---|
| 184 | + dma_addr_t context, |
|---|
| 185 | + struct safexcel_token **atoken) |
|---|
| 186 | +{ |
|---|
| 121 | 187 | struct safexcel_command_desc *cdesc; |
|---|
| 122 | | - int i; |
|---|
| 123 | 188 | |
|---|
| 124 | | - cdesc = safexcel_ring_next_wptr(priv, &priv->ring[ring_id].cdr); |
|---|
| 189 | + cdesc = safexcel_ring_next_cwptr(priv, &priv->ring[ring_id].cdr, |
|---|
| 190 | + first, atoken); |
|---|
| 125 | 191 | if (IS_ERR(cdesc)) |
|---|
| 126 | 192 | return cdesc; |
|---|
| 127 | 193 | |
|---|
| 128 | | - memset(cdesc, 0, sizeof(struct safexcel_command_desc)); |
|---|
| 129 | | - |
|---|
| 130 | | - cdesc->first_seg = first; |
|---|
| 131 | | - cdesc->last_seg = last; |
|---|
| 132 | 194 | cdesc->particle_size = data_len; |
|---|
| 195 | + cdesc->rsvd0 = 0; |
|---|
| 196 | + cdesc->last_seg = last; |
|---|
| 197 | + cdesc->first_seg = first; |
|---|
| 198 | + cdesc->additional_cdata_size = 0; |
|---|
| 199 | + cdesc->rsvd1 = 0; |
|---|
| 133 | 200 | cdesc->data_lo = lower_32_bits(data); |
|---|
| 134 | 201 | cdesc->data_hi = upper_32_bits(data); |
|---|
| 135 | 202 | |
|---|
| 136 | | - if (first && context) { |
|---|
| 137 | | - struct safexcel_token *token = |
|---|
| 138 | | - (struct safexcel_token *)cdesc->control_data.token; |
|---|
| 139 | | - |
|---|
| 140 | | - cdesc->control_data.packet_length = full_data_len; |
|---|
| 203 | + if (first) { |
|---|
| 204 | + /* |
|---|
| 205 | + * Note that the length here MUST be >0 or else the EIP(1)97 |
|---|
| 206 | + * may hang. Newer EIP197 firmware actually incorporates this |
|---|
| 207 | + * fix already, but that doesn't help the EIP97 and we may |
|---|
| 208 | + * also be running older firmware. |
|---|
| 209 | + */ |
|---|
| 210 | + cdesc->control_data.packet_length = full_data_len ?: 1; |
|---|
| 141 | 211 | cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE | |
|---|
| 142 | 212 | EIP197_OPTION_64BIT_CTX | |
|---|
| 143 | | - EIP197_OPTION_CTX_CTRL_IN_CMD; |
|---|
| 144 | | - cdesc->control_data.context_lo = |
|---|
| 145 | | - (lower_32_bits(context) & GENMASK(31, 2)) >> 2; |
|---|
| 213 | + EIP197_OPTION_CTX_CTRL_IN_CMD | |
|---|
| 214 | + EIP197_OPTION_RC_AUTO; |
|---|
| 215 | + cdesc->control_data.type = EIP197_TYPE_BCLA; |
|---|
| 216 | + cdesc->control_data.context_lo = lower_32_bits(context) | |
|---|
| 217 | + EIP197_CONTEXT_SMALL; |
|---|
| 146 | 218 | cdesc->control_data.context_hi = upper_32_bits(context); |
|---|
| 147 | | - |
|---|
| 148 | | - /* TODO: large xform HMAC with SHA-384/512 uses refresh = 3 */ |
|---|
| 149 | | - cdesc->control_data.refresh = 2; |
|---|
| 150 | | - |
|---|
| 151 | | - for (i = 0; i < EIP197_MAX_TOKENS; i++) |
|---|
| 152 | | - eip197_noop_token(&token[i]); |
|---|
| 153 | 219 | } |
|---|
| 154 | 220 | |
|---|
| 155 | 221 | return cdesc; |
|---|
| .. | .. |
|---|
| 161 | 227 | dma_addr_t data, u32 len) |
|---|
| 162 | 228 | { |
|---|
| 163 | 229 | struct safexcel_result_desc *rdesc; |
|---|
| 230 | + struct result_data_desc *rtoken; |
|---|
| 164 | 231 | |
|---|
| 165 | | - rdesc = safexcel_ring_next_wptr(priv, &priv->ring[ring_id].rdr); |
|---|
| 232 | + rdesc = safexcel_ring_next_rwptr(priv, &priv->ring[ring_id].rdr, |
|---|
| 233 | + &rtoken); |
|---|
| 166 | 234 | if (IS_ERR(rdesc)) |
|---|
| 167 | 235 | return rdesc; |
|---|
| 168 | 236 | |
|---|
| 169 | | - memset(rdesc, 0, sizeof(struct safexcel_result_desc)); |
|---|
| 170 | | - |
|---|
| 171 | | - rdesc->first_seg = first; |
|---|
| 172 | | - rdesc->last_seg = last; |
|---|
| 173 | 237 | rdesc->particle_size = len; |
|---|
| 238 | + rdesc->rsvd0 = 0; |
|---|
| 239 | + rdesc->descriptor_overflow = 1; /* assume error */ |
|---|
| 240 | + rdesc->buffer_overflow = 1; /* assume error */ |
|---|
| 241 | + rdesc->last_seg = last; |
|---|
| 242 | + rdesc->first_seg = first; |
|---|
| 243 | + rdesc->result_size = EIP197_RD64_RESULT_SIZE; |
|---|
| 244 | + rdesc->rsvd1 = 0; |
|---|
| 174 | 245 | rdesc->data_lo = lower_32_bits(data); |
|---|
| 175 | 246 | rdesc->data_hi = upper_32_bits(data); |
|---|
| 176 | 247 | |
|---|
| 248 | + /* Clear length in result token */ |
|---|
| 249 | + rtoken->packet_length = 0; |
|---|
| 250 | + /* Assume errors - HW will clear if not the case */ |
|---|
| 251 | + rtoken->error_code = 0x7fff; |
|---|
| 252 | + |
|---|
| 177 | 253 | return rdesc; |
|---|
| 178 | 254 | } |
|---|