.. | .. |
---|
33 | 33 | #define __QEDR_H__ |
---|
34 | 34 | |
---|
35 | 35 | #include <linux/pci.h> |
---|
36 | | -#include <linux/idr.h> |
---|
| 36 | +#include <linux/xarray.h> |
---|
37 | 37 | #include <rdma/ib_addr.h> |
---|
38 | 38 | #include <linux/qed/qed_if.h> |
---|
39 | 39 | #include <linux/qed/qed_chain.h> |
---|
40 | 40 | #include <linux/qed/qed_rdma_if.h> |
---|
41 | 41 | #include <linux/qed/qede_rdma.h> |
---|
42 | 42 | #include <linux/qed/roce_common.h> |
---|
| 43 | +#include <linux/completion.h> |
---|
43 | 44 | #include "qedr_hsi_rdma.h" |
---|
44 | 45 | |
---|
45 | 46 | #define QEDR_NODE_DESC "QLogic 579xx RoCE HCA" |
---|
46 | | -#define DP_NAME(dev) ((dev)->ibdev.name) |
---|
| 47 | +#define DP_NAME(_dev) dev_name(&(_dev)->ibdev.dev) |
---|
47 | 48 | #define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP) |
---|
48 | 49 | #define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE) |
---|
49 | 50 | |
---|
.. | .. |
---|
102 | 103 | u64 max_mr_size; |
---|
103 | 104 | u32 max_cqe; |
---|
104 | 105 | u32 max_mw; |
---|
105 | | - u32 max_fmr; |
---|
106 | 106 | u32 max_mr_mw_fmr_pbl; |
---|
107 | 107 | u64 max_mr_mw_fmr_size; |
---|
108 | 108 | u32 max_pd; |
---|
.. | .. |
---|
122 | 122 | }; |
---|
123 | 123 | |
---|
124 | 124 | #define QEDR_ENET_STATE_BIT (0) |
---|
125 | | - |
---|
126 | | -struct qedr_idr { |
---|
127 | | - spinlock_t idr_lock; /* Protect idr data-structure */ |
---|
128 | | - struct idr idr; |
---|
129 | | -}; |
---|
130 | 125 | |
---|
131 | 126 | struct qedr_dev { |
---|
132 | 127 | struct ib_device ibdev; |
---|
.. | .. |
---|
162 | 157 | u32 dp_module; |
---|
163 | 158 | u8 dp_level; |
---|
164 | 159 | u8 num_hwfns; |
---|
| 160 | +#define QEDR_IS_CMT(dev) ((dev)->num_hwfns > 1) |
---|
| 161 | + u8 affin_hwfn_idx; |
---|
165 | 162 | u8 gsi_ll2_handle; |
---|
166 | 163 | |
---|
167 | 164 | uint wq_multiplier; |
---|
.. | .. |
---|
171 | 168 | struct qedr_cq *gsi_rqcq; |
---|
172 | 169 | struct qedr_qp *gsi_qp; |
---|
173 | 170 | enum qed_rdma_type rdma_type; |
---|
174 | | - struct qedr_idr qpidr; |
---|
175 | | - struct qedr_idr srqidr; |
---|
| 171 | + struct xarray qps; |
---|
| 172 | + struct xarray srqs; |
---|
176 | 173 | struct workqueue_struct *iwarp_wq; |
---|
177 | 174 | u16 iwarp_max_mtu; |
---|
178 | 175 | |
---|
.. | .. |
---|
232 | 229 | struct ib_ucontext ibucontext; |
---|
233 | 230 | struct qedr_dev *dev; |
---|
234 | 231 | struct qedr_pd *pd; |
---|
235 | | - u64 dpi_addr; |
---|
| 232 | + void __iomem *dpi_addr; |
---|
| 233 | + struct rdma_user_mmap_entry *db_mmap_entry; |
---|
236 | 234 | u64 dpi_phys_addr; |
---|
237 | 235 | u32 dpi_size; |
---|
238 | 236 | u16 dpi; |
---|
| 237 | + bool db_rec; |
---|
| 238 | + u8 edpm_mode; |
---|
| 239 | +}; |
---|
239 | 240 | |
---|
240 | | - struct list_head mm_head; |
---|
241 | | - |
---|
242 | | - /* Lock to protect mm list */ |
---|
243 | | - struct mutex mm_list_lock; |
---|
| 241 | +union db_prod32 { |
---|
| 242 | + struct rdma_pwm_val16_data data; |
---|
| 243 | + u32 raw; |
---|
244 | 244 | }; |
---|
245 | 245 | |
---|
246 | 246 | union db_prod64 { |
---|
.. | .. |
---|
268 | 268 | struct qedr_pbl *pbl_tbl; |
---|
269 | 269 | u64 buf_addr; |
---|
270 | 270 | size_t buf_len; |
---|
| 271 | + |
---|
| 272 | + /* doorbell recovery */ |
---|
| 273 | + void __iomem *db_addr; |
---|
| 274 | + struct qedr_user_db_rec *db_rec_data; |
---|
| 275 | + struct rdma_user_mmap_entry *db_mmap_entry; |
---|
| 276 | + void __iomem *db_rec_db2_addr; |
---|
| 277 | + union db_prod32 db_rec_db2_data; |
---|
271 | 278 | }; |
---|
272 | 279 | |
---|
273 | 280 | struct qedr_cq { |
---|
.. | .. |
---|
303 | 310 | struct qedr_ucontext *uctx; |
---|
304 | 311 | }; |
---|
305 | 312 | |
---|
306 | | -struct qedr_mm { |
---|
307 | | - struct { |
---|
308 | | - u64 phy_addr; |
---|
309 | | - unsigned long len; |
---|
310 | | - } key; |
---|
311 | | - struct list_head entry; |
---|
312 | | -}; |
---|
313 | | - |
---|
314 | | -union db_prod32 { |
---|
315 | | - struct rdma_pwm_val16_data data; |
---|
316 | | - u32 raw; |
---|
| 313 | +struct qedr_xrcd { |
---|
| 314 | + struct ib_xrcd ibxrcd; |
---|
| 315 | + u16 xrcd_id; |
---|
317 | 316 | }; |
---|
318 | 317 | |
---|
319 | 318 | struct qedr_qp_hwq_info { |
---|
.. | .. |
---|
367 | 366 | struct ib_umem *prod_umem; |
---|
368 | 367 | u16 srq_id; |
---|
369 | 368 | u32 srq_limit; |
---|
| 369 | + bool is_xrc; |
---|
370 | 370 | /* lock to protect srq recv post */ |
---|
371 | 371 | spinlock_t lock; |
---|
372 | 372 | }; |
---|
.. | .. |
---|
380 | 380 | QEDR_QP_ERR_RQ_PBL_FULL = 32, |
---|
381 | 381 | }; |
---|
382 | 382 | |
---|
| 383 | +enum qedr_qp_create_type { |
---|
| 384 | + QEDR_QP_CREATE_NONE, |
---|
| 385 | + QEDR_QP_CREATE_USER, |
---|
| 386 | + QEDR_QP_CREATE_KERNEL, |
---|
| 387 | +}; |
---|
| 388 | + |
---|
| 389 | +enum qedr_iwarp_cm_flags { |
---|
| 390 | + QEDR_IWARP_CM_WAIT_FOR_CONNECT = BIT(0), |
---|
| 391 | + QEDR_IWARP_CM_WAIT_FOR_DISCONNECT = BIT(1), |
---|
| 392 | +}; |
---|
| 393 | + |
---|
383 | 394 | struct qedr_qp { |
---|
384 | 395 | struct ib_qp ibqp; /* must be first */ |
---|
385 | 396 | struct qedr_dev *dev; |
---|
386 | | - struct qedr_iw_ep *ep; |
---|
387 | 397 | struct qedr_qp_hwq_info sq; |
---|
388 | 398 | struct qedr_qp_hwq_info rq; |
---|
389 | 399 | |
---|
.. | .. |
---|
398 | 408 | u32 id; |
---|
399 | 409 | struct qedr_pd *pd; |
---|
400 | 410 | enum ib_qp_type qp_type; |
---|
| 411 | + enum qedr_qp_create_type create_type; |
---|
401 | 412 | struct qed_rdma_qp *qed_qp; |
---|
402 | 413 | u32 qp_id; |
---|
403 | 414 | u16 icid; |
---|
.. | .. |
---|
407 | 418 | u32 sq_psn; |
---|
408 | 419 | u32 qkey; |
---|
409 | 420 | u32 dest_qp_num; |
---|
| 421 | + u8 timeout; |
---|
410 | 422 | |
---|
411 | 423 | /* Relevant to qps created from kernel space only (ULPs) */ |
---|
412 | 424 | u8 prev_wqe_size; |
---|
.. | .. |
---|
440 | 452 | /* Relevant to qps created from user space only (applications) */ |
---|
441 | 453 | struct qedr_userq usq; |
---|
442 | 454 | struct qedr_userq urq; |
---|
443 | | - atomic_t refcnt; |
---|
444 | | - bool destroyed; |
---|
| 455 | + |
---|
| 456 | + /* synchronization objects used with iwarp ep */ |
---|
| 457 | + struct kref refcnt; |
---|
| 458 | + struct completion iwarp_cm_comp; |
---|
| 459 | + unsigned long iwarp_cm_flags; /* enum iwarp_cm_flags */ |
---|
445 | 460 | }; |
---|
446 | 461 | |
---|
447 | 462 | struct qedr_ah { |
---|
.. | .. |
---|
477 | 492 | |
---|
478 | 493 | u64 *pages; |
---|
479 | 494 | u32 npages; |
---|
| 495 | +}; |
---|
| 496 | + |
---|
| 497 | +struct qedr_user_mmap_entry { |
---|
| 498 | + struct rdma_user_mmap_entry rdma_entry; |
---|
| 499 | + struct qedr_dev *dev; |
---|
| 500 | + union { |
---|
| 501 | + u64 io_address; |
---|
| 502 | + void *address; |
---|
| 503 | + }; |
---|
| 504 | + size_t length; |
---|
| 505 | + u16 dpi; |
---|
| 506 | + u8 mmap_flag; |
---|
480 | 507 | }; |
---|
481 | 508 | |
---|
482 | 509 | #define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT))) |
---|
.. | .. |
---|
534 | 561 | struct iw_cm_id *cm_id; |
---|
535 | 562 | struct qedr_qp *qp; |
---|
536 | 563 | void *qed_context; |
---|
537 | | - u8 during_connect; |
---|
| 564 | + struct kref refcnt; |
---|
538 | 565 | }; |
---|
539 | 566 | |
---|
540 | 567 | static inline |
---|
.. | .. |
---|
551 | 578 | static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd) |
---|
552 | 579 | { |
---|
553 | 580 | return container_of(ibpd, struct qedr_pd, ibpd); |
---|
| 581 | +} |
---|
| 582 | + |
---|
| 583 | +static inline struct qedr_xrcd *get_qedr_xrcd(struct ib_xrcd *ibxrcd) |
---|
| 584 | +{ |
---|
| 585 | + return container_of(ibxrcd, struct qedr_xrcd, ibxrcd); |
---|
554 | 586 | } |
---|
555 | 587 | |
---|
556 | 588 | static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq) |
---|
.. | .. |
---|
577 | 609 | { |
---|
578 | 610 | return container_of(ibsrq, struct qedr_srq, ibsrq); |
---|
579 | 611 | } |
---|
| 612 | + |
---|
| 613 | +static inline bool qedr_qp_has_srq(struct qedr_qp *qp) |
---|
| 614 | +{ |
---|
| 615 | + return qp->srq; |
---|
| 616 | +} |
---|
| 617 | + |
---|
| 618 | +static inline bool qedr_qp_has_sq(struct qedr_qp *qp) |
---|
| 619 | +{ |
---|
| 620 | + if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_TGT) |
---|
| 621 | + return 0; |
---|
| 622 | + |
---|
| 623 | + return 1; |
---|
| 624 | +} |
---|
| 625 | + |
---|
| 626 | +static inline bool qedr_qp_has_rq(struct qedr_qp *qp) |
---|
| 627 | +{ |
---|
| 628 | + if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_INI || |
---|
| 629 | + qp->qp_type == IB_QPT_XRC_TGT || qedr_qp_has_srq(qp)) |
---|
| 630 | + return 0; |
---|
| 631 | + |
---|
| 632 | + return 1; |
---|
| 633 | +} |
---|
| 634 | + |
---|
| 635 | +static inline struct qedr_user_mmap_entry * |
---|
| 636 | +get_qedr_mmap_entry(struct rdma_user_mmap_entry *rdma_entry) |
---|
| 637 | +{ |
---|
| 638 | + return container_of(rdma_entry, struct qedr_user_mmap_entry, |
---|
| 639 | + rdma_entry); |
---|
| 640 | +} |
---|
580 | 641 | #endif |
---|