| .. | .. |
|---|
| 33 | 33 | #define __QEDR_H__ |
|---|
| 34 | 34 | |
|---|
| 35 | 35 | #include <linux/pci.h> |
|---|
| 36 | | -#include <linux/idr.h> |
|---|
| 36 | +#include <linux/xarray.h> |
|---|
| 37 | 37 | #include <rdma/ib_addr.h> |
|---|
| 38 | 38 | #include <linux/qed/qed_if.h> |
|---|
| 39 | 39 | #include <linux/qed/qed_chain.h> |
|---|
| 40 | 40 | #include <linux/qed/qed_rdma_if.h> |
|---|
| 41 | 41 | #include <linux/qed/qede_rdma.h> |
|---|
| 42 | 42 | #include <linux/qed/roce_common.h> |
|---|
| 43 | +#include <linux/completion.h> |
|---|
| 43 | 44 | #include "qedr_hsi_rdma.h" |
|---|
| 44 | 45 | |
|---|
| 45 | 46 | #define QEDR_NODE_DESC "QLogic 579xx RoCE HCA" |
|---|
| 46 | | -#define DP_NAME(dev) ((dev)->ibdev.name) |
|---|
| 47 | +#define DP_NAME(_dev) dev_name(&(_dev)->ibdev.dev) |
|---|
| 47 | 48 | #define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP) |
|---|
| 48 | 49 | #define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE) |
|---|
| 49 | 50 | |
|---|
| .. | .. |
|---|
| 102 | 103 | u64 max_mr_size; |
|---|
| 103 | 104 | u32 max_cqe; |
|---|
| 104 | 105 | u32 max_mw; |
|---|
| 105 | | - u32 max_fmr; |
|---|
| 106 | 106 | u32 max_mr_mw_fmr_pbl; |
|---|
| 107 | 107 | u64 max_mr_mw_fmr_size; |
|---|
| 108 | 108 | u32 max_pd; |
|---|
| .. | .. |
|---|
| 122 | 122 | }; |
|---|
| 123 | 123 | |
|---|
| 124 | 124 | #define QEDR_ENET_STATE_BIT (0) |
|---|
| 125 | | - |
|---|
| 126 | | -struct qedr_idr { |
|---|
| 127 | | - spinlock_t idr_lock; /* Protect idr data-structure */ |
|---|
| 128 | | - struct idr idr; |
|---|
| 129 | | -}; |
|---|
| 130 | 125 | |
|---|
| 131 | 126 | struct qedr_dev { |
|---|
| 132 | 127 | struct ib_device ibdev; |
|---|
| .. | .. |
|---|
| 162 | 157 | u32 dp_module; |
|---|
| 163 | 158 | u8 dp_level; |
|---|
| 164 | 159 | u8 num_hwfns; |
|---|
| 160 | +#define QEDR_IS_CMT(dev) ((dev)->num_hwfns > 1) |
|---|
| 161 | + u8 affin_hwfn_idx; |
|---|
| 165 | 162 | u8 gsi_ll2_handle; |
|---|
| 166 | 163 | |
|---|
| 167 | 164 | uint wq_multiplier; |
|---|
| .. | .. |
|---|
| 171 | 168 | struct qedr_cq *gsi_rqcq; |
|---|
| 172 | 169 | struct qedr_qp *gsi_qp; |
|---|
| 173 | 170 | enum qed_rdma_type rdma_type; |
|---|
| 174 | | - struct qedr_idr qpidr; |
|---|
| 175 | | - struct qedr_idr srqidr; |
|---|
| 171 | + struct xarray qps; |
|---|
| 172 | + struct xarray srqs; |
|---|
| 176 | 173 | struct workqueue_struct *iwarp_wq; |
|---|
| 177 | 174 | u16 iwarp_max_mtu; |
|---|
| 178 | 175 | |
|---|
| .. | .. |
|---|
| 232 | 229 | struct ib_ucontext ibucontext; |
|---|
| 233 | 230 | struct qedr_dev *dev; |
|---|
| 234 | 231 | struct qedr_pd *pd; |
|---|
| 235 | | - u64 dpi_addr; |
|---|
| 232 | + void __iomem *dpi_addr; |
|---|
| 233 | + struct rdma_user_mmap_entry *db_mmap_entry; |
|---|
| 236 | 234 | u64 dpi_phys_addr; |
|---|
| 237 | 235 | u32 dpi_size; |
|---|
| 238 | 236 | u16 dpi; |
|---|
| 237 | + bool db_rec; |
|---|
| 238 | + u8 edpm_mode; |
|---|
| 239 | +}; |
|---|
| 239 | 240 | |
|---|
| 240 | | - struct list_head mm_head; |
|---|
| 241 | | - |
|---|
| 242 | | - /* Lock to protect mm list */ |
|---|
| 243 | | - struct mutex mm_list_lock; |
|---|
| 241 | +union db_prod32 { |
|---|
| 242 | + struct rdma_pwm_val16_data data; |
|---|
| 243 | + u32 raw; |
|---|
| 244 | 244 | }; |
|---|
| 245 | 245 | |
|---|
| 246 | 246 | union db_prod64 { |
|---|
| .. | .. |
|---|
| 268 | 268 | struct qedr_pbl *pbl_tbl; |
|---|
| 269 | 269 | u64 buf_addr; |
|---|
| 270 | 270 | size_t buf_len; |
|---|
| 271 | + |
|---|
| 272 | + /* doorbell recovery */ |
|---|
| 273 | + void __iomem *db_addr; |
|---|
| 274 | + struct qedr_user_db_rec *db_rec_data; |
|---|
| 275 | + struct rdma_user_mmap_entry *db_mmap_entry; |
|---|
| 276 | + void __iomem *db_rec_db2_addr; |
|---|
| 277 | + union db_prod32 db_rec_db2_data; |
|---|
| 271 | 278 | }; |
|---|
| 272 | 279 | |
|---|
| 273 | 280 | struct qedr_cq { |
|---|
| .. | .. |
|---|
| 303 | 310 | struct qedr_ucontext *uctx; |
|---|
| 304 | 311 | }; |
|---|
| 305 | 312 | |
|---|
| 306 | | -struct qedr_mm { |
|---|
| 307 | | - struct { |
|---|
| 308 | | - u64 phy_addr; |
|---|
| 309 | | - unsigned long len; |
|---|
| 310 | | - } key; |
|---|
| 311 | | - struct list_head entry; |
|---|
| 312 | | -}; |
|---|
| 313 | | - |
|---|
| 314 | | -union db_prod32 { |
|---|
| 315 | | - struct rdma_pwm_val16_data data; |
|---|
| 316 | | - u32 raw; |
|---|
| 313 | +struct qedr_xrcd { |
|---|
| 314 | + struct ib_xrcd ibxrcd; |
|---|
| 315 | + u16 xrcd_id; |
|---|
| 317 | 316 | }; |
|---|
| 318 | 317 | |
|---|
| 319 | 318 | struct qedr_qp_hwq_info { |
|---|
| .. | .. |
|---|
| 367 | 366 | struct ib_umem *prod_umem; |
|---|
| 368 | 367 | u16 srq_id; |
|---|
| 369 | 368 | u32 srq_limit; |
|---|
| 369 | + bool is_xrc; |
|---|
| 370 | 370 | /* lock to protect srq recv post */ |
|---|
| 371 | 371 | spinlock_t lock; |
|---|
| 372 | 372 | }; |
|---|
| .. | .. |
|---|
| 380 | 380 | QEDR_QP_ERR_RQ_PBL_FULL = 32, |
|---|
| 381 | 381 | }; |
|---|
| 382 | 382 | |
|---|
| 383 | +enum qedr_qp_create_type { |
|---|
| 384 | + QEDR_QP_CREATE_NONE, |
|---|
| 385 | + QEDR_QP_CREATE_USER, |
|---|
| 386 | + QEDR_QP_CREATE_KERNEL, |
|---|
| 387 | +}; |
|---|
| 388 | + |
|---|
| 389 | +enum qedr_iwarp_cm_flags { |
|---|
| 390 | + QEDR_IWARP_CM_WAIT_FOR_CONNECT = BIT(0), |
|---|
| 391 | + QEDR_IWARP_CM_WAIT_FOR_DISCONNECT = BIT(1), |
|---|
| 392 | +}; |
|---|
| 393 | + |
|---|
| 383 | 394 | struct qedr_qp { |
|---|
| 384 | 395 | struct ib_qp ibqp; /* must be first */ |
|---|
| 385 | 396 | struct qedr_dev *dev; |
|---|
| 386 | | - struct qedr_iw_ep *ep; |
|---|
| 387 | 397 | struct qedr_qp_hwq_info sq; |
|---|
| 388 | 398 | struct qedr_qp_hwq_info rq; |
|---|
| 389 | 399 | |
|---|
| .. | .. |
|---|
| 398 | 408 | u32 id; |
|---|
| 399 | 409 | struct qedr_pd *pd; |
|---|
| 400 | 410 | enum ib_qp_type qp_type; |
|---|
| 411 | + enum qedr_qp_create_type create_type; |
|---|
| 401 | 412 | struct qed_rdma_qp *qed_qp; |
|---|
| 402 | 413 | u32 qp_id; |
|---|
| 403 | 414 | u16 icid; |
|---|
| .. | .. |
|---|
| 407 | 418 | u32 sq_psn; |
|---|
| 408 | 419 | u32 qkey; |
|---|
| 409 | 420 | u32 dest_qp_num; |
|---|
| 421 | + u8 timeout; |
|---|
| 410 | 422 | |
|---|
| 411 | 423 | /* Relevant to qps created from kernel space only (ULPs) */ |
|---|
| 412 | 424 | u8 prev_wqe_size; |
|---|
| .. | .. |
|---|
| 440 | 452 | /* Relevant to qps created from user space only (applications) */ |
|---|
| 441 | 453 | struct qedr_userq usq; |
|---|
| 442 | 454 | struct qedr_userq urq; |
|---|
| 443 | | - atomic_t refcnt; |
|---|
| 444 | | - bool destroyed; |
|---|
| 455 | + |
|---|
| 456 | + /* synchronization objects used with iwarp ep */ |
|---|
| 457 | + struct kref refcnt; |
|---|
| 458 | + struct completion iwarp_cm_comp; |
|---|
| 459 | + unsigned long iwarp_cm_flags; /* enum iwarp_cm_flags */ |
|---|
| 445 | 460 | }; |
|---|
| 446 | 461 | |
|---|
| 447 | 462 | struct qedr_ah { |
|---|
| .. | .. |
|---|
| 477 | 492 | |
|---|
| 478 | 493 | u64 *pages; |
|---|
| 479 | 494 | u32 npages; |
|---|
| 495 | +}; |
|---|
| 496 | + |
|---|
| 497 | +struct qedr_user_mmap_entry { |
|---|
| 498 | + struct rdma_user_mmap_entry rdma_entry; |
|---|
| 499 | + struct qedr_dev *dev; |
|---|
| 500 | + union { |
|---|
| 501 | + u64 io_address; |
|---|
| 502 | + void *address; |
|---|
| 503 | + }; |
|---|
| 504 | + size_t length; |
|---|
| 505 | + u16 dpi; |
|---|
| 506 | + u8 mmap_flag; |
|---|
| 480 | 507 | }; |
|---|
| 481 | 508 | |
|---|
| 482 | 509 | #define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT))) |
|---|
| .. | .. |
|---|
| 534 | 561 | struct iw_cm_id *cm_id; |
|---|
| 535 | 562 | struct qedr_qp *qp; |
|---|
| 536 | 563 | void *qed_context; |
|---|
| 537 | | - u8 during_connect; |
|---|
| 564 | + struct kref refcnt; |
|---|
| 538 | 565 | }; |
|---|
| 539 | 566 | |
|---|
| 540 | 567 | static inline |
|---|
| .. | .. |
|---|
| 551 | 578 | static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd) |
|---|
| 552 | 579 | { |
|---|
| 553 | 580 | return container_of(ibpd, struct qedr_pd, ibpd); |
|---|
| 581 | +} |
|---|
| 582 | + |
|---|
| 583 | +static inline struct qedr_xrcd *get_qedr_xrcd(struct ib_xrcd *ibxrcd) |
|---|
| 584 | +{ |
|---|
| 585 | + return container_of(ibxrcd, struct qedr_xrcd, ibxrcd); |
|---|
| 554 | 586 | } |
|---|
| 555 | 587 | |
|---|
| 556 | 588 | static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq) |
|---|
| .. | .. |
|---|
| 577 | 609 | { |
|---|
| 578 | 610 | return container_of(ibsrq, struct qedr_srq, ibsrq); |
|---|
| 579 | 611 | } |
|---|
| 612 | + |
|---|
| 613 | +static inline bool qedr_qp_has_srq(struct qedr_qp *qp) |
|---|
| 614 | +{ |
|---|
| 615 | + return qp->srq; |
|---|
| 616 | +} |
|---|
| 617 | + |
|---|
| 618 | +static inline bool qedr_qp_has_sq(struct qedr_qp *qp) |
|---|
| 619 | +{ |
|---|
| 620 | + if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_TGT) |
|---|
| 621 | + return 0; |
|---|
| 622 | + |
|---|
| 623 | + return 1; |
|---|
| 624 | +} |
|---|
| 625 | + |
|---|
| 626 | +static inline bool qedr_qp_has_rq(struct qedr_qp *qp) |
|---|
| 627 | +{ |
|---|
| 628 | + if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_XRC_INI || |
|---|
| 629 | + qp->qp_type == IB_QPT_XRC_TGT || qedr_qp_has_srq(qp)) |
|---|
| 630 | + return 0; |
|---|
| 631 | + |
|---|
| 632 | + return 1; |
|---|
| 633 | +} |
|---|
| 634 | + |
|---|
| 635 | +static inline struct qedr_user_mmap_entry * |
|---|
| 636 | +get_qedr_mmap_entry(struct rdma_user_mmap_entry *rdma_entry) |
|---|
| 637 | +{ |
|---|
| 638 | + return container_of(rdma_entry, struct qedr_user_mmap_entry, |
|---|
| 639 | + rdma_entry); |
|---|
| 640 | +} |
|---|
| 580 | 641 | #endif |
|---|