| .. | .. |
|---|
| 38 | 38 | |
|---|
| 39 | 39 | #include "mlx4_ib.h" |
|---|
| 40 | 40 | #include <rdma/mlx4-abi.h> |
|---|
| 41 | +#include <rdma/uverbs_ioctl.h> |
|---|
| 41 | 42 | |
|---|
| 42 | 43 | static void mlx4_ib_cq_comp(struct mlx4_cq *cq) |
|---|
| 43 | 44 | { |
|---|
| .. | .. |
|---|
| 134 | 135 | mlx4_buf_free(dev->dev, (cqe + 1) * buf->entry_size, &buf->buf); |
|---|
| 135 | 136 | } |
|---|
| 136 | 137 | |
|---|
| 137 | | -static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_ucontext *context, |
|---|
| 138 | | - struct mlx4_ib_cq_buf *buf, struct ib_umem **umem, |
|---|
| 139 | | - u64 buf_addr, int cqe) |
|---|
| 138 | +static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_udata *udata, |
|---|
| 139 | + struct mlx4_ib_cq_buf *buf, |
|---|
| 140 | + struct ib_umem **umem, u64 buf_addr, int cqe) |
|---|
| 140 | 141 | { |
|---|
| 141 | 142 | int err; |
|---|
| 142 | 143 | int cqe_size = dev->dev->caps.cqe_size; |
|---|
| 143 | 144 | int shift; |
|---|
| 144 | 145 | int n; |
|---|
| 145 | 146 | |
|---|
| 146 | | - *umem = ib_umem_get(context, buf_addr, cqe * cqe_size, |
|---|
| 147 | | - IB_ACCESS_LOCAL_WRITE, 1); |
|---|
| 147 | + *umem = ib_umem_get(&dev->ib_dev, buf_addr, cqe * cqe_size, |
|---|
| 148 | + IB_ACCESS_LOCAL_WRITE); |
|---|
| 148 | 149 | if (IS_ERR(*umem)) |
|---|
| 149 | 150 | return PTR_ERR(*umem); |
|---|
| 150 | 151 | |
|---|
| 151 | | - n = ib_umem_page_count(*umem); |
|---|
| 152 | 152 | shift = mlx4_ib_umem_calc_optimal_mtt_size(*umem, 0, &n); |
|---|
| 153 | 153 | err = mlx4_mtt_init(dev->dev, n, shift, &buf->mtt); |
|---|
| 154 | 154 | |
|---|
| .. | .. |
|---|
| 171 | 171 | } |
|---|
| 172 | 172 | |
|---|
| 173 | 173 | #define CQ_CREATE_FLAGS_SUPPORTED IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION |
|---|
| 174 | | -struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, |
|---|
| 175 | | - const struct ib_cq_init_attr *attr, |
|---|
| 176 | | - struct ib_ucontext *context, |
|---|
| 177 | | - struct ib_udata *udata) |
|---|
| 174 | +int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, |
|---|
| 175 | + struct ib_udata *udata) |
|---|
| 178 | 176 | { |
|---|
| 177 | + struct ib_device *ibdev = ibcq->device; |
|---|
| 179 | 178 | int entries = attr->cqe; |
|---|
| 180 | 179 | int vector = attr->comp_vector; |
|---|
| 181 | 180 | struct mlx4_ib_dev *dev = to_mdev(ibdev); |
|---|
| 182 | | - struct mlx4_ib_cq *cq; |
|---|
| 181 | + struct mlx4_ib_cq *cq = to_mcq(ibcq); |
|---|
| 183 | 182 | struct mlx4_uar *uar; |
|---|
| 183 | + void *buf_addr; |
|---|
| 184 | 184 | int err; |
|---|
| 185 | + struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context( |
|---|
| 186 | + udata, struct mlx4_ib_ucontext, ibucontext); |
|---|
| 185 | 187 | |
|---|
| 186 | 188 | if (entries < 1 || entries > dev->dev->caps.max_cqes) |
|---|
| 187 | | - return ERR_PTR(-EINVAL); |
|---|
| 189 | + return -EINVAL; |
|---|
| 188 | 190 | |
|---|
| 189 | 191 | if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED) |
|---|
| 190 | | - return ERR_PTR(-EINVAL); |
|---|
| 191 | | - |
|---|
| 192 | | - cq = kmalloc(sizeof *cq, GFP_KERNEL); |
|---|
| 193 | | - if (!cq) |
|---|
| 194 | | - return ERR_PTR(-ENOMEM); |
|---|
| 192 | + return -EINVAL; |
|---|
| 195 | 193 | |
|---|
| 196 | 194 | entries = roundup_pow_of_two(entries + 1); |
|---|
| 197 | 195 | cq->ibcq.cqe = entries - 1; |
|---|
| .. | .. |
|---|
| 203 | 201 | INIT_LIST_HEAD(&cq->send_qp_list); |
|---|
| 204 | 202 | INIT_LIST_HEAD(&cq->recv_qp_list); |
|---|
| 205 | 203 | |
|---|
| 206 | | - if (context) { |
|---|
| 204 | + if (udata) { |
|---|
| 207 | 205 | struct mlx4_ib_create_cq ucmd; |
|---|
| 208 | 206 | |
|---|
| 209 | 207 | if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { |
|---|
| .. | .. |
|---|
| 211 | 209 | goto err_cq; |
|---|
| 212 | 210 | } |
|---|
| 213 | 211 | |
|---|
| 214 | | - err = mlx4_ib_get_cq_umem(dev, context, &cq->buf, &cq->umem, |
|---|
| 212 | + buf_addr = (void *)(unsigned long)ucmd.buf_addr; |
|---|
| 213 | + err = mlx4_ib_get_cq_umem(dev, udata, &cq->buf, &cq->umem, |
|---|
| 215 | 214 | ucmd.buf_addr, entries); |
|---|
| 216 | 215 | if (err) |
|---|
| 217 | 216 | goto err_cq; |
|---|
| 218 | 217 | |
|---|
| 219 | | - err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr, |
|---|
| 220 | | - &cq->db); |
|---|
| 218 | + err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &cq->db); |
|---|
| 221 | 219 | if (err) |
|---|
| 222 | 220 | goto err_mtt; |
|---|
| 223 | 221 | |
|---|
| 224 | | - uar = &to_mucontext(context)->uar; |
|---|
| 222 | + uar = &context->uar; |
|---|
| 225 | 223 | cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS; |
|---|
| 226 | 224 | } else { |
|---|
| 227 | 225 | err = mlx4_db_alloc(dev->dev, &cq->db, 1); |
|---|
| .. | .. |
|---|
| 237 | 235 | if (err) |
|---|
| 238 | 236 | goto err_db; |
|---|
| 239 | 237 | |
|---|
| 238 | + buf_addr = &cq->buf.buf; |
|---|
| 239 | + |
|---|
| 240 | 240 | uar = &dev->priv_uar; |
|---|
| 241 | 241 | cq->mcq.usage = MLX4_RES_USAGE_DRIVER; |
|---|
| 242 | 242 | } |
|---|
| .. | .. |
|---|
| 244 | 244 | if (dev->eq_table) |
|---|
| 245 | 245 | vector = dev->eq_table[vector % ibdev->num_comp_vectors]; |
|---|
| 246 | 246 | |
|---|
| 247 | | - err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, |
|---|
| 248 | | - cq->db.dma, &cq->mcq, vector, 0, |
|---|
| 249 | | - !!(cq->create_flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION)); |
|---|
| 247 | + err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, cq->db.dma, |
|---|
| 248 | + &cq->mcq, vector, 0, |
|---|
| 249 | + !!(cq->create_flags & |
|---|
| 250 | + IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION), |
|---|
| 251 | + buf_addr, !!udata); |
|---|
| 250 | 252 | if (err) |
|---|
| 251 | 253 | goto err_dbmap; |
|---|
| 252 | 254 | |
|---|
| 253 | | - if (context) |
|---|
| 255 | + if (udata) |
|---|
| 254 | 256 | cq->mcq.tasklet_ctx.comp = mlx4_ib_cq_comp; |
|---|
| 255 | 257 | else |
|---|
| 256 | 258 | cq->mcq.comp = mlx4_ib_cq_comp; |
|---|
| 257 | 259 | cq->mcq.event = mlx4_ib_cq_event; |
|---|
| 258 | 260 | |
|---|
| 259 | | - if (context) |
|---|
| 261 | + if (udata) |
|---|
| 260 | 262 | if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) { |
|---|
| 261 | 263 | err = -EFAULT; |
|---|
| 262 | 264 | goto err_cq_free; |
|---|
| 263 | 265 | } |
|---|
| 264 | 266 | |
|---|
| 265 | | - return &cq->ibcq; |
|---|
| 267 | + return 0; |
|---|
| 266 | 268 | |
|---|
| 267 | 269 | err_cq_free: |
|---|
| 268 | 270 | mlx4_cq_free(dev->dev, &cq->mcq); |
|---|
| 269 | 271 | |
|---|
| 270 | 272 | err_dbmap: |
|---|
| 271 | | - if (context) |
|---|
| 272 | | - mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db); |
|---|
| 273 | + if (udata) |
|---|
| 274 | + mlx4_ib_db_unmap_user(context, &cq->db); |
|---|
| 273 | 275 | |
|---|
| 274 | 276 | err_mtt: |
|---|
| 275 | 277 | mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); |
|---|
| 276 | 278 | |
|---|
| 277 | | - if (context) |
|---|
| 278 | | - ib_umem_release(cq->umem); |
|---|
| 279 | | - else |
|---|
| 279 | + ib_umem_release(cq->umem); |
|---|
| 280 | + if (!udata) |
|---|
| 280 | 281 | mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); |
|---|
| 281 | 282 | |
|---|
| 282 | 283 | err_db: |
|---|
| 283 | | - if (!context) |
|---|
| 284 | + if (!udata) |
|---|
| 284 | 285 | mlx4_db_free(dev->dev, &cq->db); |
|---|
| 285 | | - |
|---|
| 286 | 286 | err_cq: |
|---|
| 287 | | - kfree(cq); |
|---|
| 288 | | - |
|---|
| 289 | | - return ERR_PTR(err); |
|---|
| 287 | + return err; |
|---|
| 290 | 288 | } |
|---|
| 291 | 289 | |
|---|
| 292 | 290 | static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, |
|---|
| .. | .. |
|---|
| 329 | 327 | if (!cq->resize_buf) |
|---|
| 330 | 328 | return -ENOMEM; |
|---|
| 331 | 329 | |
|---|
| 332 | | - err = mlx4_ib_get_cq_umem(dev, cq->umem->context, &cq->resize_buf->buf, |
|---|
| 330 | + err = mlx4_ib_get_cq_umem(dev, udata, &cq->resize_buf->buf, |
|---|
| 333 | 331 | &cq->resize_umem, ucmd.buf_addr, entries); |
|---|
| 334 | 332 | if (err) { |
|---|
| 335 | 333 | kfree(cq->resize_buf); |
|---|
| .. | .. |
|---|
| 468 | 466 | kfree(cq->resize_buf); |
|---|
| 469 | 467 | cq->resize_buf = NULL; |
|---|
| 470 | 468 | |
|---|
| 471 | | - if (cq->resize_umem) { |
|---|
| 472 | | - ib_umem_release(cq->resize_umem); |
|---|
| 473 | | - cq->resize_umem = NULL; |
|---|
| 474 | | - } |
|---|
| 475 | | - |
|---|
| 469 | + ib_umem_release(cq->resize_umem); |
|---|
| 470 | + cq->resize_umem = NULL; |
|---|
| 476 | 471 | out: |
|---|
| 477 | 472 | mutex_unlock(&cq->resize_mutex); |
|---|
| 478 | 473 | |
|---|
| 479 | 474 | return err; |
|---|
| 480 | 475 | } |
|---|
| 481 | 476 | |
|---|
| 482 | | -int mlx4_ib_destroy_cq(struct ib_cq *cq) |
|---|
| 477 | +int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) |
|---|
| 483 | 478 | { |
|---|
| 484 | 479 | struct mlx4_ib_dev *dev = to_mdev(cq->device); |
|---|
| 485 | 480 | struct mlx4_ib_cq *mcq = to_mcq(cq); |
|---|
| .. | .. |
|---|
| 487 | 482 | mlx4_cq_free(dev->dev, &mcq->mcq); |
|---|
| 488 | 483 | mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt); |
|---|
| 489 | 484 | |
|---|
| 490 | | - if (cq->uobject) { |
|---|
| 491 | | - mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db); |
|---|
| 492 | | - ib_umem_release(mcq->umem); |
|---|
| 485 | + if (udata) { |
|---|
| 486 | + mlx4_ib_db_unmap_user( |
|---|
| 487 | + rdma_udata_to_drv_context( |
|---|
| 488 | + udata, |
|---|
| 489 | + struct mlx4_ib_ucontext, |
|---|
| 490 | + ibucontext), |
|---|
| 491 | + &mcq->db); |
|---|
| 493 | 492 | } else { |
|---|
| 494 | 493 | mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); |
|---|
| 495 | 494 | mlx4_db_free(dev->dev, &mcq->db); |
|---|
| 496 | 495 | } |
|---|
| 497 | | - |
|---|
| 498 | | - kfree(mcq); |
|---|
| 499 | | - |
|---|
| 496 | + ib_umem_release(mcq->umem); |
|---|
| 500 | 497 | return 0; |
|---|
| 501 | 498 | } |
|---|
| 502 | 499 | |
|---|
| .. | .. |
|---|
| 571 | 568 | wc->vendor_err = cqe->vendor_err_syndrome; |
|---|
| 572 | 569 | } |
|---|
| 573 | 570 | |
|---|
| 574 | | -static int mlx4_ib_ipoib_csum_ok(__be16 status, __be16 checksum) |
|---|
| 571 | +static int mlx4_ib_ipoib_csum_ok(__be16 status, u8 badfcs_enc, __be16 checksum) |
|---|
| 575 | 572 | { |
|---|
| 576 | | - return ((status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 | |
|---|
| 577 | | - MLX4_CQE_STATUS_IPV4F | |
|---|
| 578 | | - MLX4_CQE_STATUS_IPV4OPT | |
|---|
| 579 | | - MLX4_CQE_STATUS_IPV6 | |
|---|
| 580 | | - MLX4_CQE_STATUS_IPOK)) == |
|---|
| 581 | | - cpu_to_be16(MLX4_CQE_STATUS_IPV4 | |
|---|
| 582 | | - MLX4_CQE_STATUS_IPOK)) && |
|---|
| 583 | | - (status & cpu_to_be16(MLX4_CQE_STATUS_UDP | |
|---|
| 584 | | - MLX4_CQE_STATUS_TCP)) && |
|---|
| 585 | | - checksum == cpu_to_be16(0xffff); |
|---|
| 573 | + return ((badfcs_enc & MLX4_CQE_STATUS_L4_CSUM) || |
|---|
| 574 | + ((status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && |
|---|
| 575 | + (status & cpu_to_be16(MLX4_CQE_STATUS_TCP | |
|---|
| 576 | + MLX4_CQE_STATUS_UDP)) && |
|---|
| 577 | + (checksum == cpu_to_be16(0xffff)))); |
|---|
| 586 | 578 | } |
|---|
| 587 | 579 | |
|---|
| 588 | 580 | static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc, |
|---|
| .. | .. |
|---|
| 773 | 765 | switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { |
|---|
| 774 | 766 | case MLX4_OPCODE_RDMA_WRITE_IMM: |
|---|
| 775 | 767 | wc->wc_flags |= IB_WC_WITH_IMM; |
|---|
| 776 | | - /* fall through */ |
|---|
| 768 | + fallthrough; |
|---|
| 777 | 769 | case MLX4_OPCODE_RDMA_WRITE: |
|---|
| 778 | 770 | wc->opcode = IB_WC_RDMA_WRITE; |
|---|
| 779 | 771 | break; |
|---|
| 780 | 772 | case MLX4_OPCODE_SEND_IMM: |
|---|
| 781 | 773 | wc->wc_flags |= IB_WC_WITH_IMM; |
|---|
| 782 | | - /* fall through */ |
|---|
| 774 | + fallthrough; |
|---|
| 783 | 775 | case MLX4_OPCODE_SEND: |
|---|
| 784 | 776 | case MLX4_OPCODE_SEND_INVAL: |
|---|
| 785 | 777 | wc->opcode = IB_WC_SEND; |
|---|
| .. | .. |
|---|
| 858 | 850 | wc->wc_flags |= g_mlpath_rqpn & 0x80000000 ? IB_WC_GRH : 0; |
|---|
| 859 | 851 | wc->pkey_index = be32_to_cpu(cqe->immed_rss_invalid) & 0x7f; |
|---|
| 860 | 852 | wc->wc_flags |= mlx4_ib_ipoib_csum_ok(cqe->status, |
|---|
| 853 | + cqe->badfcs_enc, |
|---|
| 861 | 854 | cqe->checksum) ? IB_WC_IP_CSUM_OK : 0; |
|---|
| 862 | 855 | if (is_eth) { |
|---|
| 863 | 856 | wc->slid = 0; |
|---|