.. | .. |
---|
40 | 40 | #include <linux/skbuff.h> |
---|
41 | 41 | #include <linux/inetdevice.h> |
---|
42 | 42 | #include <linux/atomic.h> |
---|
| 43 | +#include <net/tls.h> |
---|
43 | 44 | #include "cxgb4.h" |
---|
44 | 45 | |
---|
45 | 46 | #define MAX_ULD_QSETS 16 |
---|
| 47 | +#define MAX_ULD_NPORTS 4 |
---|
46 | 48 | |
---|
47 | 49 | /* ulp_mem_io + ulptx_idata + payload + padding */ |
---|
48 | 50 | #define MAX_IMM_ULPTX_WR_LEN (32 + 8 + 256 + 8) |
---|
.. | .. |
---|
92 | 94 | union aopen_entry *next; |
---|
93 | 95 | }; |
---|
94 | 96 | |
---|
| 97 | +struct eotid_entry { |
---|
| 98 | + void *data; |
---|
| 99 | +}; |
---|
| 100 | + |
---|
95 | 101 | /* |
---|
96 | 102 | * Holds the size, base address, free list start, etc of the TID, server TID, |
---|
97 | 103 | * and active-open TID tables. The tables themselves are allocated dynamically. |
---|
98 | 104 | */ |
---|
99 | 105 | struct tid_info { |
---|
100 | 106 | void **tid_tab; |
---|
| 107 | + unsigned int tid_base; |
---|
101 | 108 | unsigned int ntids; |
---|
102 | 109 | |
---|
103 | 110 | struct serv_entry *stid_tab; |
---|
104 | 111 | unsigned long *stid_bmap; |
---|
105 | 112 | unsigned int nstids; |
---|
106 | 113 | unsigned int stid_base; |
---|
| 114 | + |
---|
| 115 | + unsigned int nhash; |
---|
107 | 116 | unsigned int hash_base; |
---|
108 | 117 | |
---|
109 | 118 | union aopen_entry *atid_tab; |
---|
110 | 119 | unsigned int natids; |
---|
111 | 120 | unsigned int atid_base; |
---|
| 121 | + |
---|
| 122 | + struct filter_entry *hpftid_tab; |
---|
| 123 | + unsigned long *hpftid_bmap; |
---|
| 124 | + unsigned int nhpftids; |
---|
| 125 | + unsigned int hpftid_base; |
---|
112 | 126 | |
---|
113 | 127 | struct filter_entry *ftid_tab; |
---|
114 | 128 | unsigned long *ftid_bmap; |
---|
.. | .. |
---|
129 | 143 | unsigned int v6_stids_in_use; |
---|
130 | 144 | unsigned int sftids_in_use; |
---|
131 | 145 | |
---|
| 146 | + /* ETHOFLD range */ |
---|
| 147 | + struct eotid_entry *eotid_tab; |
---|
| 148 | + unsigned long *eotid_bmap; |
---|
| 149 | + unsigned int eotid_base; |
---|
| 150 | + unsigned int neotids; |
---|
| 151 | + |
---|
132 | 152 | /* TIDs in the TCAM */ |
---|
133 | 153 | atomic_t tids_in_use; |
---|
134 | 154 | /* TIDs in the HASH */ |
---|
135 | 155 | atomic_t hash_tids_in_use; |
---|
136 | 156 | atomic_t conns_in_use; |
---|
| 157 | + /* ETHOFLD TIDs used for rate limiting */ |
---|
| 158 | + atomic_t eotids_in_use; |
---|
| 159 | + |
---|
137 | 160 | /* lock for setting/clearing filter bitmap */ |
---|
138 | 161 | spinlock_t ftid_lock; |
---|
| 162 | + |
---|
| 163 | + unsigned int tc_hash_tids_max_prio; |
---|
139 | 164 | }; |
---|
140 | 165 | |
---|
141 | 166 | static inline void *lookup_tid(const struct tid_info *t, unsigned int tid) |
---|
142 | 167 | { |
---|
| 168 | + tid -= t->tid_base; |
---|
143 | 169 | return tid < t->ntids ? t->tid_tab[tid] : NULL; |
---|
| 170 | +} |
---|
| 171 | + |
---|
| 172 | +static inline bool tid_out_of_range(const struct tid_info *t, unsigned int tid) |
---|
| 173 | +{ |
---|
| 174 | + return ((tid - t->tid_base) >= t->ntids); |
---|
144 | 175 | } |
---|
145 | 176 | |
---|
146 | 177 | static inline void *lookup_atid(const struct tid_info *t, unsigned int atid) |
---|
.. | .. |
---|
164 | 195 | static inline void cxgb4_insert_tid(struct tid_info *t, void *data, |
---|
165 | 196 | unsigned int tid, unsigned short family) |
---|
166 | 197 | { |
---|
167 | | - t->tid_tab[tid] = data; |
---|
| 198 | + t->tid_tab[tid - t->tid_base] = data; |
---|
168 | 199 | if (t->hash_base && (tid >= t->hash_base)) { |
---|
169 | 200 | if (family == AF_INET6) |
---|
170 | 201 | atomic_add(2, &t->hash_tids_in_use); |
---|
.. | .. |
---|
177 | 208 | atomic_inc(&t->tids_in_use); |
---|
178 | 209 | } |
---|
179 | 210 | atomic_inc(&t->conns_in_use); |
---|
| 211 | +} |
---|
| 212 | + |
---|
| 213 | +static inline struct eotid_entry *cxgb4_lookup_eotid(struct tid_info *t, |
---|
| 214 | + u32 eotid) |
---|
| 215 | +{ |
---|
| 216 | + return eotid < t->neotids ? &t->eotid_tab[eotid] : NULL; |
---|
| 217 | +} |
---|
| 218 | + |
---|
| 219 | +static inline int cxgb4_get_free_eotid(struct tid_info *t) |
---|
| 220 | +{ |
---|
| 221 | + int eotid; |
---|
| 222 | + |
---|
| 223 | + eotid = find_first_zero_bit(t->eotid_bmap, t->neotids); |
---|
| 224 | + if (eotid >= t->neotids) |
---|
| 225 | + eotid = -1; |
---|
| 226 | + |
---|
| 227 | + return eotid; |
---|
| 228 | +} |
---|
| 229 | + |
---|
| 230 | +static inline void cxgb4_alloc_eotid(struct tid_info *t, u32 eotid, void *data) |
---|
| 231 | +{ |
---|
| 232 | + set_bit(eotid, t->eotid_bmap); |
---|
| 233 | + t->eotid_tab[eotid].data = data; |
---|
| 234 | + atomic_inc(&t->eotids_in_use); |
---|
| 235 | +} |
---|
| 236 | + |
---|
| 237 | +static inline void cxgb4_free_eotid(struct tid_info *t, u32 eotid) |
---|
| 238 | +{ |
---|
| 239 | + clear_bit(eotid, t->eotid_bmap); |
---|
| 240 | + t->eotid_tab[eotid].data = NULL; |
---|
| 241 | + atomic_dec(&t->eotids_in_use); |
---|
180 | 242 | } |
---|
181 | 243 | |
---|
182 | 244 | int cxgb4_alloc_atid(struct tid_info *t, void *data); |
---|
.. | .. |
---|
213 | 275 | u32 tid; /* to store tid */ |
---|
214 | 276 | }; |
---|
215 | 277 | |
---|
| 278 | +struct chcr_ktls { |
---|
| 279 | + refcount_t ktls_refcount; |
---|
| 280 | +}; |
---|
| 281 | + |
---|
216 | 282 | struct ch_filter_specification; |
---|
217 | 283 | |
---|
218 | | -int cxgb4_get_free_ftid(struct net_device *dev, int family); |
---|
| 284 | +int cxgb4_get_free_ftid(struct net_device *dev, u8 family, bool hash_en, |
---|
| 285 | + u32 tc_prio); |
---|
219 | 286 | int __cxgb4_set_filter(struct net_device *dev, int filter_id, |
---|
220 | 287 | struct ch_filter_specification *fs, |
---|
221 | 288 | struct filter_ctx *ctx); |
---|
.. | .. |
---|
240 | 307 | CXGB4_ULD_ISCSI, |
---|
241 | 308 | CXGB4_ULD_ISCSIT, |
---|
242 | 309 | CXGB4_ULD_CRYPTO, |
---|
| 310 | + CXGB4_ULD_IPSEC, |
---|
243 | 311 | CXGB4_ULD_TLS, |
---|
| 312 | + CXGB4_ULD_KTLS, |
---|
244 | 313 | CXGB4_ULD_MAX |
---|
245 | 314 | }; |
---|
246 | 315 | |
---|
.. | .. |
---|
271 | 340 | CXGB4_CONTROL_DB_DROP, |
---|
272 | 341 | }; |
---|
273 | 342 | |
---|
| 343 | +struct adapter; |
---|
274 | 344 | struct pci_dev; |
---|
275 | 345 | struct l2t_data; |
---|
276 | 346 | struct net_device; |
---|
.. | .. |
---|
295 | 365 | struct cxgb4_range ocq; |
---|
296 | 366 | struct cxgb4_range key; |
---|
297 | 367 | unsigned int ncrypto_fc; |
---|
| 368 | + struct cxgb4_range ppod_edram; |
---|
298 | 369 | }; |
---|
| 370 | + |
---|
| 371 | +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) |
---|
| 372 | +struct ch_ktls_port_stats_debug { |
---|
| 373 | + atomic64_t ktls_tx_connection_open; |
---|
| 374 | + atomic64_t ktls_tx_connection_fail; |
---|
| 375 | + atomic64_t ktls_tx_connection_close; |
---|
| 376 | + atomic64_t ktls_tx_encrypted_packets; |
---|
| 377 | + atomic64_t ktls_tx_encrypted_bytes; |
---|
| 378 | + atomic64_t ktls_tx_ctx; |
---|
| 379 | + atomic64_t ktls_tx_ooo; |
---|
| 380 | + atomic64_t ktls_tx_skip_no_sync_data; |
---|
| 381 | + atomic64_t ktls_tx_drop_no_sync_data; |
---|
| 382 | + atomic64_t ktls_tx_drop_bypass_req; |
---|
| 383 | +}; |
---|
| 384 | + |
---|
| 385 | +struct ch_ktls_stats_debug { |
---|
| 386 | + struct ch_ktls_port_stats_debug ktls_port[MAX_ULD_NPORTS]; |
---|
| 387 | + atomic64_t ktls_tx_send_records; |
---|
| 388 | + atomic64_t ktls_tx_end_pkts; |
---|
| 389 | + atomic64_t ktls_tx_start_pkts; |
---|
| 390 | + atomic64_t ktls_tx_middle_pkts; |
---|
| 391 | + atomic64_t ktls_tx_retransmit_pkts; |
---|
| 392 | + atomic64_t ktls_tx_complete_pkts; |
---|
| 393 | + atomic64_t ktls_tx_trimmed_pkts; |
---|
| 394 | + atomic64_t ktls_tx_fallback; |
---|
| 395 | +}; |
---|
| 396 | +#endif |
---|
299 | 397 | |
---|
300 | 398 | struct chcr_stats_debug { |
---|
301 | 399 | atomic_t cipher_rqst; |
---|
.. | .. |
---|
304 | 402 | atomic_t complete; |
---|
305 | 403 | atomic_t error; |
---|
306 | 404 | atomic_t fallback; |
---|
307 | | - atomic_t ipsec_cnt; |
---|
308 | 405 | atomic_t tls_pdu_tx; |
---|
309 | 406 | atomic_t tls_pdu_rx; |
---|
310 | 407 | atomic_t tls_key; |
---|
311 | 408 | }; |
---|
| 409 | + |
---|
| 410 | +#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) |
---|
| 411 | +struct ch_ipsec_stats_debug { |
---|
| 412 | + atomic_t ipsec_cnt; |
---|
| 413 | +}; |
---|
| 414 | +#endif |
---|
312 | 415 | |
---|
313 | 416 | #define OCQ_WIN_OFFSET(pdev, vres) \ |
---|
314 | 417 | (pci_resource_len((pdev), 2) - roundup_pow_of_two((vres)->ocq.size)) |
---|
.. | .. |
---|
339 | 442 | unsigned int cclk_ps; /* Core clock period in psec */ |
---|
340 | 443 | unsigned short udb_density; /* # of user DB/page */ |
---|
341 | 444 | unsigned short ucq_density; /* # of user CQs/page */ |
---|
| 445 | + unsigned int sge_host_page_size; /* SGE host page size */ |
---|
342 | 446 | unsigned short filt_mode; /* filter optional components */ |
---|
343 | 447 | unsigned short tx_modq[NCHAN]; /* maps each tx channel to a */ |
---|
344 | 448 | /* scheduler queue */ |
---|
.. | .. |
---|
385 | 489 | struct napi_struct *napi); |
---|
386 | 490 | void (*lro_flush)(struct t4_lro_mgr *); |
---|
387 | 491 | int (*tx_handler)(struct sk_buff *skb, struct net_device *dev); |
---|
| 492 | +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) |
---|
| 493 | + const struct tlsdev_ops *tlsdev_ops; |
---|
| 494 | +#endif |
---|
| 495 | +#if IS_ENABLED(CONFIG_XFRM_OFFLOAD) |
---|
| 496 | + const struct xfrmdev_ops *xfrmdev_ops; |
---|
| 497 | +#endif |
---|
388 | 498 | }; |
---|
389 | 499 | |
---|
390 | | -int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p); |
---|
| 500 | +static inline bool cxgb4_is_ktls_skb(struct sk_buff *skb) |
---|
| 501 | +{ |
---|
| 502 | + return skb->sk && tls_is_sk_tx_device_offloaded(skb->sk); |
---|
| 503 | +} |
---|
| 504 | + |
---|
| 505 | +void cxgb4_uld_enable(struct adapter *adap); |
---|
| 506 | +void cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p); |
---|
391 | 507 | int cxgb4_unregister_uld(enum cxgb4_uld type); |
---|
392 | 508 | int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb); |
---|
393 | 509 | int cxgb4_immdata_send(struct net_device *dev, unsigned int idx, |
---|
.. | .. |
---|
395 | 511 | int cxgb4_crypto_send(struct net_device *dev, struct sk_buff *skb); |
---|
396 | 512 | unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo); |
---|
397 | 513 | unsigned int cxgb4_port_chan(const struct net_device *dev); |
---|
| 514 | +unsigned int cxgb4_port_e2cchan(const struct net_device *dev); |
---|
398 | 515 | unsigned int cxgb4_port_viid(const struct net_device *dev); |
---|
399 | 516 | unsigned int cxgb4_tp_smt_idx(enum chip_type chip, unsigned int viid); |
---|
400 | 517 | unsigned int cxgb4_port_idx(const struct net_device *dev); |
---|