.. | .. |
---|
43 | 43 | #define RDS_MR_8K_SCALE (256 / (RDS_MR_8K_MSG_SIZE + 1)) |
---|
44 | 44 | #define RDS_MR_8K_POOL_SIZE (RDS_MR_8K_SCALE * (8192 / 2)) |
---|
45 | 45 | |
---|
46 | | -struct rds_ib_fmr { |
---|
47 | | - struct ib_fmr *fmr; |
---|
48 | | -}; |
---|
49 | | - |
---|
50 | 46 | enum rds_ib_fr_state { |
---|
51 | 47 | FRMR_IS_FREE, /* mr invalidated & ready for use */ |
---|
52 | 48 | FRMR_IS_INUSE, /* mr is in use or used & can be invalidated */ |
---|
.. | .. |
---|
57 | 53 | struct ib_mr *mr; |
---|
58 | 54 | enum rds_ib_fr_state fr_state; |
---|
59 | 55 | bool fr_inv; |
---|
| 56 | + wait_queue_head_t fr_inv_done; |
---|
| 57 | + bool fr_reg; |
---|
| 58 | + wait_queue_head_t fr_reg_done; |
---|
60 | 59 | struct ib_send_wr fr_wr; |
---|
61 | 60 | unsigned int dma_npages; |
---|
62 | 61 | unsigned int sg_byte_len; |
---|
.. | .. |
---|
64 | 63 | |
---|
65 | 64 | /* This is stored as mr->r_trans_private. */ |
---|
66 | 65 | struct rds_ib_mr { |
---|
| 66 | + struct delayed_work work; |
---|
67 | 67 | struct rds_ib_device *device; |
---|
68 | 68 | struct rds_ib_mr_pool *pool; |
---|
69 | 69 | struct rds_ib_connection *ic; |
---|
.. | .. |
---|
78 | 78 | unsigned int sg_len; |
---|
79 | 79 | int sg_dma_len; |
---|
80 | 80 | |
---|
| 81 | + u8 odp:1; |
---|
81 | 82 | union { |
---|
82 | | - struct rds_ib_fmr fmr; |
---|
83 | 83 | struct rds_ib_frmr frmr; |
---|
| 84 | + struct ib_mr *mr; |
---|
84 | 85 | } u; |
---|
85 | 86 | }; |
---|
86 | 87 | |
---|
.. | .. |
---|
97 | 98 | struct llist_head free_list; /* unused MRs */ |
---|
98 | 99 | struct llist_head clean_list; /* unused & unmapped MRs */ |
---|
99 | 100 | wait_queue_head_t flush_wait; |
---|
| 101 | + spinlock_t clean_lock; /* "clean_list" concurrency */ |
---|
100 | 102 | |
---|
101 | 103 | atomic_t free_pinned; /* memory pinned by free MRs */ |
---|
102 | 104 | unsigned long max_items; |
---|
103 | 105 | unsigned long max_items_soft; |
---|
104 | 106 | unsigned long max_free_pinned; |
---|
105 | | - struct ib_fmr_attr fmr_attr; |
---|
106 | | - bool use_fastreg; |
---|
| 107 | + unsigned int max_pages; |
---|
107 | 108 | }; |
---|
108 | 109 | |
---|
109 | 110 | extern struct workqueue_struct *rds_ib_mr_wq; |
---|
.. | .. |
---|
118 | 119 | void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *); |
---|
119 | 120 | void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents, |
---|
120 | 121 | struct rds_sock *rs, u32 *key_ret, |
---|
121 | | - struct rds_connection *conn); |
---|
| 122 | + struct rds_connection *conn, u64 start, u64 length, |
---|
| 123 | + int need_odp); |
---|
122 | 124 | void rds_ib_sync_mr(void *trans_private, int dir); |
---|
123 | 125 | void rds_ib_free_mr(void *trans_private, int invalidate); |
---|
124 | 126 | void rds_ib_flush_mrs(void); |
---|
125 | 127 | int rds_ib_mr_init(void); |
---|
126 | 128 | void rds_ib_mr_exit(void); |
---|
| 129 | +u32 rds_ib_get_lkey(void *trans_private); |
---|
127 | 130 | |
---|
128 | 131 | void __rds_ib_teardown_mr(struct rds_ib_mr *); |
---|
129 | 132 | void rds_ib_teardown_mr(struct rds_ib_mr *); |
---|
130 | | -struct rds_ib_mr *rds_ib_alloc_fmr(struct rds_ib_device *, int); |
---|
131 | 133 | struct rds_ib_mr *rds_ib_reuse_mr(struct rds_ib_mr_pool *); |
---|
132 | 134 | int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *, int, struct rds_ib_mr **); |
---|
133 | | -struct rds_ib_mr *rds_ib_reg_fmr(struct rds_ib_device *, struct scatterlist *, |
---|
134 | | - unsigned long, u32 *); |
---|
135 | 135 | struct rds_ib_mr *rds_ib_try_reuse_ibmr(struct rds_ib_mr_pool *); |
---|
136 | | -void rds_ib_unreg_fmr(struct list_head *, unsigned int *, |
---|
137 | | - unsigned long *, unsigned int); |
---|
138 | | -void rds_ib_free_fmr_list(struct rds_ib_mr *); |
---|
139 | 136 | struct rds_ib_mr *rds_ib_reg_frmr(struct rds_ib_device *rds_ibdev, |
---|
140 | 137 | struct rds_ib_connection *ic, |
---|
141 | 138 | struct scatterlist *sg, |
---|