| .. | .. |
|---|
| 34 | 34 | #include <linux/platform_device.h> |
|---|
| 35 | 35 | #include <linux/vmalloc.h> |
|---|
| 36 | 36 | #include "hns_roce_device.h" |
|---|
| 37 | +#include <rdma/ib_umem.h> |
|---|
| 37 | 38 | |
|---|
| 38 | 39 | int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj) |
|---|
| 39 | 40 | { |
|---|
| .. | .. |
|---|
| 54 | 55 | bitmap->last = 0; |
|---|
| 55 | 56 | *obj |= bitmap->top; |
|---|
| 56 | 57 | } else { |
|---|
| 57 | | - ret = -1; |
|---|
| 58 | + ret = -EINVAL; |
|---|
| 58 | 59 | } |
|---|
| 59 | 60 | |
|---|
| 60 | 61 | spin_unlock(&bitmap->lock); |
|---|
| .. | .. |
|---|
| 67 | 68 | { |
|---|
| 68 | 69 | hns_roce_bitmap_free_range(bitmap, obj, 1, rr); |
|---|
| 69 | 70 | } |
|---|
| 70 | | -EXPORT_SYMBOL_GPL(hns_roce_bitmap_free); |
|---|
| 71 | 71 | |
|---|
| 72 | 72 | int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt, |
|---|
| 73 | 73 | int align, unsigned long *obj) |
|---|
| .. | .. |
|---|
| 100 | 100 | } |
|---|
| 101 | 101 | *obj |= bitmap->top; |
|---|
| 102 | 102 | } else { |
|---|
| 103 | | - ret = -1; |
|---|
| 103 | + ret = -EINVAL; |
|---|
| 104 | 104 | } |
|---|
| 105 | 105 | |
|---|
| 106 | 106 | spin_unlock(&bitmap->lock); |
|---|
| .. | .. |
|---|
| 157 | 157 | kfree(bitmap->table); |
|---|
| 158 | 158 | } |
|---|
| 159 | 159 | |
|---|
| 160 | | -void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size, |
|---|
| 161 | | - struct hns_roce_buf *buf) |
|---|
| 160 | +void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf) |
|---|
| 162 | 161 | { |
|---|
| 163 | | - int i; |
|---|
| 164 | 162 | struct device *dev = hr_dev->dev; |
|---|
| 163 | + u32 size = buf->size; |
|---|
| 164 | + int i; |
|---|
| 165 | 165 | |
|---|
| 166 | | - if (buf->nbufs == 1) { |
|---|
| 166 | + if (size == 0) |
|---|
| 167 | + return; |
|---|
| 168 | + |
|---|
| 169 | + buf->size = 0; |
|---|
| 170 | + |
|---|
| 171 | + if (hns_roce_buf_is_direct(buf)) { |
|---|
| 167 | 172 | dma_free_coherent(dev, size, buf->direct.buf, buf->direct.map); |
|---|
| 168 | 173 | } else { |
|---|
| 169 | | - for (i = 0; i < buf->nbufs; ++i) |
|---|
| 174 | + for (i = 0; i < buf->npages; ++i) |
|---|
| 170 | 175 | if (buf->page_list[i].buf) |
|---|
| 171 | 176 | dma_free_coherent(dev, 1 << buf->page_shift, |
|---|
| 172 | 177 | buf->page_list[i].buf, |
|---|
| 173 | 178 | buf->page_list[i].map); |
|---|
| 174 | 179 | kfree(buf->page_list); |
|---|
| 180 | + buf->page_list = NULL; |
|---|
| 175 | 181 | } |
|---|
| 176 | 182 | } |
|---|
| 177 | | -EXPORT_SYMBOL_GPL(hns_roce_buf_free); |
|---|
| 178 | 183 | |
|---|
| 179 | 184 | int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, |
|---|
| 180 | 185 | struct hns_roce_buf *buf, u32 page_shift) |
|---|
| 181 | 186 | { |
|---|
| 182 | | - int i = 0; |
|---|
| 183 | | - dma_addr_t t; |
|---|
| 187 | + struct hns_roce_buf_list *buf_list; |
|---|
| 184 | 188 | struct device *dev = hr_dev->dev; |
|---|
| 185 | | - u32 page_size = 1 << page_shift; |
|---|
| 186 | | - u32 order; |
|---|
| 189 | + u32 page_size; |
|---|
| 190 | + int i; |
|---|
| 187 | 191 | |
|---|
| 188 | | - /* SQ/RQ buf lease than one page, SQ + RQ = 8K */ |
|---|
| 192 | + /* The minimum shift of the page accessed by hw is HNS_HW_PAGE_SHIFT */ |
|---|
| 193 | + buf->page_shift = max_t(int, HNS_HW_PAGE_SHIFT, page_shift); |
|---|
| 194 | + |
|---|
| 195 | + page_size = 1 << buf->page_shift; |
|---|
| 196 | + buf->npages = DIV_ROUND_UP(size, page_size); |
|---|
| 197 | + |
|---|
| 198 | + /* required size is not bigger than one trunk size */ |
|---|
| 189 | 199 | if (size <= max_direct) { |
|---|
| 190 | | - buf->nbufs = 1; |
|---|
| 191 | | - /* Npages calculated by page_size */ |
|---|
| 192 | | - order = get_order(size); |
|---|
| 193 | | - if (order <= page_shift - PAGE_SHIFT) |
|---|
| 194 | | - order = 0; |
|---|
| 195 | | - else |
|---|
| 196 | | - order -= page_shift - PAGE_SHIFT; |
|---|
| 197 | | - buf->npages = 1 << order; |
|---|
| 198 | | - buf->page_shift = page_shift; |
|---|
| 199 | | - /* MTT PA must be recorded in 4k alignment, t is 4k aligned */ |
|---|
| 200 | | - buf->direct.buf = dma_zalloc_coherent(dev, |
|---|
| 201 | | - size, &t, GFP_KERNEL); |
|---|
| 200 | + buf->page_list = NULL; |
|---|
| 201 | + buf->direct.buf = dma_alloc_coherent(dev, size, |
|---|
| 202 | + &buf->direct.map, |
|---|
| 203 | + GFP_KERNEL); |
|---|
| 202 | 204 | if (!buf->direct.buf) |
|---|
| 203 | 205 | return -ENOMEM; |
|---|
| 204 | | - |
|---|
| 205 | | - buf->direct.map = t; |
|---|
| 206 | | - |
|---|
| 207 | | - while (t & ((1 << buf->page_shift) - 1)) { |
|---|
| 208 | | - --buf->page_shift; |
|---|
| 209 | | - buf->npages *= 2; |
|---|
| 210 | | - } |
|---|
| 211 | 206 | } else { |
|---|
| 212 | | - buf->nbufs = (size + page_size - 1) / page_size; |
|---|
| 213 | | - buf->npages = buf->nbufs; |
|---|
| 214 | | - buf->page_shift = page_shift; |
|---|
| 215 | | - buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), |
|---|
| 216 | | - GFP_KERNEL); |
|---|
| 217 | | - |
|---|
| 218 | | - if (!buf->page_list) |
|---|
| 207 | + buf_list = kcalloc(buf->npages, sizeof(*buf_list), GFP_KERNEL); |
|---|
| 208 | + if (!buf_list) |
|---|
| 219 | 209 | return -ENOMEM; |
|---|
| 220 | 210 | |
|---|
| 221 | | - for (i = 0; i < buf->nbufs; ++i) { |
|---|
| 222 | | - buf->page_list[i].buf = dma_zalloc_coherent(dev, |
|---|
| 223 | | - page_size, &t, |
|---|
| 224 | | - GFP_KERNEL); |
|---|
| 225 | | - |
|---|
| 226 | | - if (!buf->page_list[i].buf) |
|---|
| 227 | | - goto err_free; |
|---|
| 228 | | - |
|---|
| 229 | | - buf->page_list[i].map = t; |
|---|
| 211 | + for (i = 0; i < buf->npages; i++) { |
|---|
| 212 | + buf_list[i].buf = dma_alloc_coherent(dev, page_size, |
|---|
| 213 | + &buf_list[i].map, |
|---|
| 214 | + GFP_KERNEL); |
|---|
| 215 | + if (!buf_list[i].buf) |
|---|
| 216 | + break; |
|---|
| 230 | 217 | } |
|---|
| 218 | + |
|---|
| 219 | + if (i != buf->npages && i > 0) { |
|---|
| 220 | + while (i-- > 0) |
|---|
| 221 | + dma_free_coherent(dev, page_size, |
|---|
| 222 | + buf_list[i].buf, |
|---|
| 223 | + buf_list[i].map); |
|---|
| 224 | + kfree(buf_list); |
|---|
| 225 | + return -ENOMEM; |
|---|
| 226 | + } |
|---|
| 227 | + buf->page_list = buf_list; |
|---|
| 231 | 228 | } |
|---|
| 229 | + buf->size = size; |
|---|
| 232 | 230 | |
|---|
| 233 | 231 | return 0; |
|---|
| 232 | +} |
|---|
| 234 | 233 | |
|---|
| 235 | | -err_free: |
|---|
| 236 | | - hns_roce_buf_free(hr_dev, size, buf); |
|---|
| 237 | | - return -ENOMEM; |
|---|
| 234 | +int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, |
|---|
| 235 | + int buf_cnt, int start, struct hns_roce_buf *buf) |
|---|
| 236 | +{ |
|---|
| 237 | + int i, end; |
|---|
| 238 | + int total; |
|---|
| 239 | + |
|---|
| 240 | + end = start + buf_cnt; |
|---|
| 241 | + if (end > buf->npages) { |
|---|
| 242 | + dev_err(hr_dev->dev, |
|---|
| 243 | + "failed to check kmem bufs, end %d + %d total %u!\n", |
|---|
| 244 | + start, buf_cnt, buf->npages); |
|---|
| 245 | + return -EINVAL; |
|---|
| 246 | + } |
|---|
| 247 | + |
|---|
| 248 | + total = 0; |
|---|
| 249 | + for (i = start; i < end; i++) |
|---|
| 250 | + bufs[total++] = hns_roce_buf_page(buf, i); |
|---|
| 251 | + |
|---|
| 252 | + return total; |
|---|
| 253 | +} |
|---|
| 254 | + |
|---|
| 255 | +int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, |
|---|
| 256 | + int buf_cnt, int start, struct ib_umem *umem, |
|---|
| 257 | + unsigned int page_shift) |
|---|
| 258 | +{ |
|---|
| 259 | + struct ib_block_iter biter; |
|---|
| 260 | + int total = 0; |
|---|
| 261 | + int idx = 0; |
|---|
| 262 | + u64 addr; |
|---|
| 263 | + |
|---|
| 264 | + if (page_shift < HNS_HW_PAGE_SHIFT) { |
|---|
| 265 | + dev_err(hr_dev->dev, "failed to check umem page shift %u!\n", |
|---|
| 266 | + page_shift); |
|---|
| 267 | + return -EINVAL; |
|---|
| 268 | + } |
|---|
| 269 | + |
|---|
| 270 | + /* convert system page cnt to hw page cnt */ |
|---|
| 271 | + rdma_umem_for_each_dma_block(umem, &biter, 1 << page_shift) { |
|---|
| 272 | + addr = rdma_block_iter_dma_address(&biter); |
|---|
| 273 | + if (idx >= start) { |
|---|
| 274 | + bufs[total++] = addr; |
|---|
| 275 | + if (total >= buf_cnt) |
|---|
| 276 | + goto done; |
|---|
| 277 | + } |
|---|
| 278 | + idx++; |
|---|
| 279 | + } |
|---|
| 280 | + |
|---|
| 281 | +done: |
|---|
| 282 | + return total; |
|---|
| 238 | 283 | } |
|---|
| 239 | 284 | |
|---|
| 240 | 285 | void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev) |
|---|
| 241 | 286 | { |
|---|
| 287 | + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) |
|---|
| 288 | + hns_roce_cleanup_srq_table(hr_dev); |
|---|
| 242 | 289 | hns_roce_cleanup_qp_table(hr_dev); |
|---|
| 243 | 290 | hns_roce_cleanup_cq_table(hr_dev); |
|---|
| 244 | 291 | hns_roce_cleanup_mr_table(hr_dev); |
|---|