.. | .. |
---|
56 | 56 | size_t size, dma_addr_t *dma_handle, |
---|
57 | 57 | int node) |
---|
58 | 58 | { |
---|
| 59 | + struct device *device = mlx5_core_dma_dev(dev); |
---|
59 | 60 | struct mlx5_priv *priv = &dev->priv; |
---|
60 | 61 | int original_node; |
---|
61 | 62 | void *cpu_handle; |
---|
62 | 63 | |
---|
63 | 64 | mutex_lock(&priv->alloc_mutex); |
---|
64 | | - original_node = dev_to_node(&dev->pdev->dev); |
---|
65 | | - set_dev_node(&dev->pdev->dev, node); |
---|
66 | | - cpu_handle = dma_zalloc_coherent(&dev->pdev->dev, size, |
---|
67 | | - dma_handle, GFP_KERNEL); |
---|
68 | | - set_dev_node(&dev->pdev->dev, original_node); |
---|
| 65 | + original_node = dev_to_node(device); |
---|
| 66 | + set_dev_node(device, node); |
---|
| 67 | + cpu_handle = dma_alloc_coherent(device, size, dma_handle, |
---|
| 68 | + GFP_KERNEL); |
---|
| 69 | + set_dev_node(device, original_node); |
---|
69 | 70 | mutex_unlock(&priv->alloc_mutex); |
---|
70 | 71 | return cpu_handle; |
---|
71 | 72 | } |
---|
72 | 73 | |
---|
73 | | -int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, |
---|
74 | | - struct mlx5_frag_buf *buf, int node) |
---|
| 74 | +static int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, |
---|
| 75 | + struct mlx5_frag_buf *buf, int node) |
---|
75 | 76 | { |
---|
76 | 77 | dma_addr_t t; |
---|
77 | 78 | |
---|
.. | .. |
---|
110 | 111 | |
---|
111 | 112 | void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf) |
---|
112 | 113 | { |
---|
113 | | - dma_free_coherent(&dev->pdev->dev, buf->size, buf->frags->buf, |
---|
| 114 | + dma_free_coherent(mlx5_core_dma_dev(dev), buf->size, buf->frags->buf, |
---|
114 | 115 | buf->frags->map); |
---|
115 | 116 | |
---|
116 | 117 | kfree(buf->frags); |
---|
.. | .. |
---|
139 | 140 | if (!frag->buf) |
---|
140 | 141 | goto err_free_buf; |
---|
141 | 142 | if (frag->map & ((1 << buf->page_shift) - 1)) { |
---|
142 | | - dma_free_coherent(&dev->pdev->dev, frag_sz, |
---|
| 143 | + dma_free_coherent(mlx5_core_dma_dev(dev), frag_sz, |
---|
143 | 144 | buf->frags[i].buf, buf->frags[i].map); |
---|
144 | 145 | mlx5_core_warn(dev, "unexpected map alignment: %pad, page_shift=%d\n", |
---|
145 | 146 | &frag->map, buf->page_shift); |
---|
.. | .. |
---|
152 | 153 | |
---|
153 | 154 | err_free_buf: |
---|
154 | 155 | while (i--) |
---|
155 | | - dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf->frags[i].buf, |
---|
| 156 | + dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE, buf->frags[i].buf, |
---|
156 | 157 | buf->frags[i].map); |
---|
157 | 158 | kfree(buf->frags); |
---|
158 | 159 | err_out: |
---|
.. | .. |
---|
168 | 169 | for (i = 0; i < buf->npages; i++) { |
---|
169 | 170 | int frag_sz = min_t(int, size, PAGE_SIZE); |
---|
170 | 171 | |
---|
171 | | - dma_free_coherent(&dev->pdev->dev, frag_sz, buf->frags[i].buf, |
---|
| 172 | + dma_free_coherent(mlx5_core_dma_dev(dev), frag_sz, buf->frags[i].buf, |
---|
172 | 173 | buf->frags[i].map); |
---|
173 | 174 | size -= frag_sz; |
---|
174 | 175 | } |
---|
.. | .. |
---|
186 | 187 | if (!pgdir) |
---|
187 | 188 | return NULL; |
---|
188 | 189 | |
---|
189 | | - pgdir->bitmap = kcalloc(BITS_TO_LONGS(db_per_page), |
---|
190 | | - sizeof(unsigned long), |
---|
191 | | - GFP_KERNEL); |
---|
192 | | - |
---|
| 190 | + pgdir->bitmap = bitmap_zalloc(db_per_page, GFP_KERNEL); |
---|
193 | 191 | if (!pgdir->bitmap) { |
---|
194 | 192 | kfree(pgdir); |
---|
195 | 193 | return NULL; |
---|
.. | .. |
---|
200 | 198 | pgdir->db_page = mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE, |
---|
201 | 199 | &pgdir->db_dma, node); |
---|
202 | 200 | if (!pgdir->db_page) { |
---|
203 | | - kfree(pgdir->bitmap); |
---|
| 201 | + bitmap_free(pgdir->bitmap); |
---|
204 | 202 | kfree(pgdir); |
---|
205 | 203 | return NULL; |
---|
206 | 204 | } |
---|
.. | .. |
---|
277 | 275 | __set_bit(db->index, db->u.pgdir->bitmap); |
---|
278 | 276 | |
---|
279 | 277 | if (bitmap_full(db->u.pgdir->bitmap, db_per_page)) { |
---|
280 | | - dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, |
---|
| 278 | + dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE, |
---|
281 | 279 | db->u.pgdir->db_page, db->u.pgdir->db_dma); |
---|
282 | 280 | list_del(&db->u.pgdir->list); |
---|
283 | | - kfree(db->u.pgdir->bitmap); |
---|
| 281 | + bitmap_free(db->u.pgdir->bitmap); |
---|
284 | 282 | kfree(db->u.pgdir); |
---|
285 | 283 | } |
---|
286 | 284 | |
---|
.. | .. |
---|
301 | 299 | } |
---|
302 | 300 | EXPORT_SYMBOL_GPL(mlx5_fill_page_array); |
---|
303 | 301 | |
---|
304 | | -void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas) |
---|
| 302 | +void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm) |
---|
305 | 303 | { |
---|
306 | 304 | int i; |
---|
307 | 305 | |
---|
| 306 | + WARN_ON(perm & 0xfc); |
---|
308 | 307 | for (i = 0; i < buf->npages; i++) |
---|
309 | | - pas[i] = cpu_to_be64(buf->frags[i].map); |
---|
| 308 | + pas[i] = cpu_to_be64(buf->frags[i].map | perm); |
---|
| 309 | +} |
---|
| 310 | +EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array_perm); |
---|
| 311 | + |
---|
| 312 | +void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas) |
---|
| 313 | +{ |
---|
| 314 | + mlx5_fill_page_frag_array_perm(buf, pas, 0); |
---|
310 | 315 | } |
---|
311 | 316 | EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array); |
---|