| .. | .. |
|---|
| 43 | 43 | #define RDS_MR_8K_SCALE (256 / (RDS_MR_8K_MSG_SIZE + 1)) |
|---|
| 44 | 44 | #define RDS_MR_8K_POOL_SIZE (RDS_MR_8K_SCALE * (8192 / 2)) |
|---|
| 45 | 45 | |
|---|
| 46 | | -struct rds_ib_fmr { |
|---|
| 47 | | - struct ib_fmr *fmr; |
|---|
| 48 | | -}; |
|---|
| 49 | | - |
|---|
| 50 | 46 | enum rds_ib_fr_state { |
|---|
| 51 | 47 | FRMR_IS_FREE, /* mr invalidated & ready for use */ |
|---|
| 52 | 48 | FRMR_IS_INUSE, /* mr is in use or used & can be invalidated */ |
|---|
| .. | .. |
|---|
| 57 | 53 | struct ib_mr *mr; |
|---|
| 58 | 54 | enum rds_ib_fr_state fr_state; |
|---|
| 59 | 55 | bool fr_inv; |
|---|
| 56 | + wait_queue_head_t fr_inv_done; |
|---|
| 57 | + bool fr_reg; |
|---|
| 58 | + wait_queue_head_t fr_reg_done; |
|---|
| 60 | 59 | struct ib_send_wr fr_wr; |
|---|
| 61 | 60 | unsigned int dma_npages; |
|---|
| 62 | 61 | unsigned int sg_byte_len; |
|---|
| .. | .. |
|---|
| 64 | 63 | |
|---|
| 65 | 64 | /* This is stored as mr->r_trans_private. */ |
|---|
| 66 | 65 | struct rds_ib_mr { |
|---|
| 66 | + struct delayed_work work; |
|---|
| 67 | 67 | struct rds_ib_device *device; |
|---|
| 68 | 68 | struct rds_ib_mr_pool *pool; |
|---|
| 69 | 69 | struct rds_ib_connection *ic; |
|---|
| .. | .. |
|---|
| 78 | 78 | unsigned int sg_len; |
|---|
| 79 | 79 | int sg_dma_len; |
|---|
| 80 | 80 | |
|---|
| 81 | + u8 odp:1; |
|---|
| 81 | 82 | union { |
|---|
| 82 | | - struct rds_ib_fmr fmr; |
|---|
| 83 | 83 | struct rds_ib_frmr frmr; |
|---|
| 84 | + struct ib_mr *mr; |
|---|
| 84 | 85 | } u; |
|---|
| 85 | 86 | }; |
|---|
| 86 | 87 | |
|---|
| .. | .. |
|---|
| 97 | 98 | struct llist_head free_list; /* unused MRs */ |
|---|
| 98 | 99 | struct llist_head clean_list; /* unused & unmapped MRs */ |
|---|
| 99 | 100 | wait_queue_head_t flush_wait; |
|---|
| 101 | + spinlock_t clean_lock; /* "clean_list" concurrency */ |
|---|
| 100 | 102 | |
|---|
| 101 | 103 | atomic_t free_pinned; /* memory pinned by free MRs */ |
|---|
| 102 | 104 | unsigned long max_items; |
|---|
| 103 | 105 | unsigned long max_items_soft; |
|---|
| 104 | 106 | unsigned long max_free_pinned; |
|---|
| 105 | | - struct ib_fmr_attr fmr_attr; |
|---|
| 106 | | - bool use_fastreg; |
|---|
| 107 | + unsigned int max_pages; |
|---|
| 107 | 108 | }; |
|---|
| 108 | 109 | |
|---|
| 109 | 110 | extern struct workqueue_struct *rds_ib_mr_wq; |
|---|
| .. | .. |
|---|
| 118 | 119 | void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *); |
|---|
| 119 | 120 | void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents, |
|---|
| 120 | 121 | struct rds_sock *rs, u32 *key_ret, |
|---|
| 121 | | - struct rds_connection *conn); |
|---|
| 122 | + struct rds_connection *conn, u64 start, u64 length, |
|---|
| 123 | + int need_odp); |
|---|
| 122 | 124 | void rds_ib_sync_mr(void *trans_private, int dir); |
|---|
| 123 | 125 | void rds_ib_free_mr(void *trans_private, int invalidate); |
|---|
| 124 | 126 | void rds_ib_flush_mrs(void); |
|---|
| 125 | 127 | int rds_ib_mr_init(void); |
|---|
| 126 | 128 | void rds_ib_mr_exit(void); |
|---|
| 129 | +u32 rds_ib_get_lkey(void *trans_private); |
|---|
| 127 | 130 | |
|---|
| 128 | 131 | void __rds_ib_teardown_mr(struct rds_ib_mr *); |
|---|
| 129 | 132 | void rds_ib_teardown_mr(struct rds_ib_mr *); |
|---|
| 130 | | -struct rds_ib_mr *rds_ib_alloc_fmr(struct rds_ib_device *, int); |
|---|
| 131 | 133 | struct rds_ib_mr *rds_ib_reuse_mr(struct rds_ib_mr_pool *); |
|---|
| 132 | 134 | int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *, int, struct rds_ib_mr **); |
|---|
| 133 | | -struct rds_ib_mr *rds_ib_reg_fmr(struct rds_ib_device *, struct scatterlist *, |
|---|
| 134 | | - unsigned long, u32 *); |
|---|
| 135 | 135 | struct rds_ib_mr *rds_ib_try_reuse_ibmr(struct rds_ib_mr_pool *); |
|---|
| 136 | | -void rds_ib_unreg_fmr(struct list_head *, unsigned int *, |
|---|
| 137 | | - unsigned long *, unsigned int); |
|---|
| 138 | | -void rds_ib_free_fmr_list(struct rds_ib_mr *); |
|---|
| 139 | 136 | struct rds_ib_mr *rds_ib_reg_frmr(struct rds_ib_device *rds_ibdev, |
|---|
| 140 | 137 | struct rds_ib_connection *ic, |
|---|
| 141 | 138 | struct scatterlist *sg, |
|---|