| .. | .. |
|---|
| 3 | 3 | #include <linux/fs.h> |
|---|
| 4 | 4 | #include <linux/poll.h> |
|---|
| 5 | 5 | #include <linux/sched/signal.h> |
|---|
| 6 | +#include <linux/eventfd.h> |
|---|
| 6 | 7 | #include <linux/uaccess.h> |
|---|
| 7 | 8 | #include <uapi/misc/ocxl.h> |
|---|
| 8 | 9 | #include <asm/reg.h> |
|---|
| .. | .. |
|---|
| 17 | 18 | static struct mutex minors_idr_lock; |
|---|
| 18 | 19 | static struct idr minors_idr; |
|---|
| 19 | 20 | |
|---|
| 20 | | -static struct ocxl_afu *find_and_get_afu(dev_t devno) |
|---|
| 21 | +static struct ocxl_file_info *find_and_get_file_info(dev_t devno) |
|---|
| 21 | 22 | { |
|---|
| 22 | | - struct ocxl_afu *afu; |
|---|
| 23 | | - int afu_minor; |
|---|
| 23 | + struct ocxl_file_info *info; |
|---|
| 24 | 24 | |
|---|
| 25 | | - afu_minor = MINOR(devno); |
|---|
| 26 | | - /* |
|---|
| 27 | | - * We don't declare an RCU critical section here, as our AFU |
|---|
| 28 | | - * is protected by a reference counter on the device. By the time the |
|---|
| 29 | | - * minor number of a device is removed from the idr, the ref count of |
|---|
| 30 | | - * the device is already at 0, so no user API will access that AFU and |
|---|
| 31 | | - * this function can't return it. |
|---|
| 32 | | - */ |
|---|
| 33 | | - afu = idr_find(&minors_idr, afu_minor); |
|---|
| 34 | | - if (afu) |
|---|
| 35 | | - ocxl_afu_get(afu); |
|---|
| 36 | | - return afu; |
|---|
| 25 | + mutex_lock(&minors_idr_lock); |
|---|
| 26 | + info = idr_find(&minors_idr, MINOR(devno)); |
|---|
| 27 | + if (info) |
|---|
| 28 | + get_device(&info->dev); |
|---|
| 29 | + mutex_unlock(&minors_idr_lock); |
|---|
| 30 | + return info; |
|---|
| 37 | 31 | } |
|---|
| 38 | 32 | |
|---|
| 39 | | -static int allocate_afu_minor(struct ocxl_afu *afu) |
|---|
| 33 | +static int allocate_minor(struct ocxl_file_info *info) |
|---|
| 40 | 34 | { |
|---|
| 41 | 35 | int minor; |
|---|
| 42 | 36 | |
|---|
| 43 | 37 | mutex_lock(&minors_idr_lock); |
|---|
| 44 | | - minor = idr_alloc(&minors_idr, afu, 0, OCXL_NUM_MINORS, GFP_KERNEL); |
|---|
| 38 | + minor = idr_alloc(&minors_idr, info, 0, OCXL_NUM_MINORS, GFP_KERNEL); |
|---|
| 45 | 39 | mutex_unlock(&minors_idr_lock); |
|---|
| 46 | 40 | return minor; |
|---|
| 47 | 41 | } |
|---|
| 48 | 42 | |
|---|
| 49 | | -static void free_afu_minor(struct ocxl_afu *afu) |
|---|
| 43 | +static void free_minor(struct ocxl_file_info *info) |
|---|
| 50 | 44 | { |
|---|
| 51 | 45 | mutex_lock(&minors_idr_lock); |
|---|
| 52 | | - idr_remove(&minors_idr, MINOR(afu->dev.devt)); |
|---|
| 46 | + idr_remove(&minors_idr, MINOR(info->dev.devt)); |
|---|
| 53 | 47 | mutex_unlock(&minors_idr_lock); |
|---|
| 54 | 48 | } |
|---|
| 55 | 49 | |
|---|
| 56 | 50 | static int afu_open(struct inode *inode, struct file *file) |
|---|
| 57 | 51 | { |
|---|
| 58 | | - struct ocxl_afu *afu; |
|---|
| 52 | + struct ocxl_file_info *info; |
|---|
| 59 | 53 | struct ocxl_context *ctx; |
|---|
| 60 | 54 | int rc; |
|---|
| 61 | 55 | |
|---|
| 62 | 56 | pr_debug("%s for device %x\n", __func__, inode->i_rdev); |
|---|
| 63 | 57 | |
|---|
| 64 | | - afu = find_and_get_afu(inode->i_rdev); |
|---|
| 65 | | - if (!afu) |
|---|
| 58 | + info = find_and_get_file_info(inode->i_rdev); |
|---|
| 59 | + if (!info) |
|---|
| 66 | 60 | return -ENODEV; |
|---|
| 67 | 61 | |
|---|
| 68 | | - ctx = ocxl_context_alloc(); |
|---|
| 69 | | - if (!ctx) { |
|---|
| 70 | | - rc = -ENOMEM; |
|---|
| 71 | | - goto put_afu; |
|---|
| 62 | + rc = ocxl_context_alloc(&ctx, info->afu, inode->i_mapping); |
|---|
| 63 | + if (rc) { |
|---|
| 64 | + put_device(&info->dev); |
|---|
| 65 | + return rc; |
|---|
| 72 | 66 | } |
|---|
| 73 | | - |
|---|
| 74 | | - rc = ocxl_context_init(ctx, afu, inode->i_mapping); |
|---|
| 75 | | - if (rc) |
|---|
| 76 | | - goto put_afu; |
|---|
| 67 | + put_device(&info->dev); |
|---|
| 77 | 68 | file->private_data = ctx; |
|---|
| 78 | | - ocxl_afu_put(afu); |
|---|
| 79 | 69 | return 0; |
|---|
| 80 | | - |
|---|
| 81 | | -put_afu: |
|---|
| 82 | | - ocxl_afu_put(afu); |
|---|
| 83 | | - return rc; |
|---|
| 84 | 70 | } |
|---|
| 85 | 71 | |
|---|
| 86 | 72 | static long afu_ioctl_attach(struct ocxl_context *ctx, |
|---|
| .. | .. |
|---|
| 100 | 86 | return -EINVAL; |
|---|
| 101 | 87 | |
|---|
| 102 | 88 | amr = arg.amr & mfspr(SPRN_UAMOR); |
|---|
| 103 | | - rc = ocxl_context_attach(ctx, amr); |
|---|
| 89 | + rc = ocxl_context_attach(ctx, amr, current->mm); |
|---|
| 104 | 90 | return rc; |
|---|
| 105 | 91 | } |
|---|
| 106 | 92 | |
|---|
| .. | .. |
|---|
| 151 | 137 | mutex_unlock(&ctx->status_mutex); |
|---|
| 152 | 138 | |
|---|
| 153 | 139 | if (status == ATTACHED) { |
|---|
| 154 | | - int rc; |
|---|
| 155 | | - struct link *link = ctx->afu->fn->link; |
|---|
| 140 | + int rc = ocxl_link_update_pe(ctx->afu->fn->link, |
|---|
| 141 | + ctx->pasid, ctx->tidr); |
|---|
| 156 | 142 | |
|---|
| 157 | | - rc = ocxl_link_update_pe(link, ctx->pasid, ctx->tidr); |
|---|
| 158 | 143 | if (rc) |
|---|
| 159 | 144 | return rc; |
|---|
| 160 | 145 | } |
|---|
| .. | .. |
|---|
| 198 | 183 | x == OCXL_IOCTL_GET_FEATURES ? "GET_FEATURES" : \ |
|---|
| 199 | 184 | "UNKNOWN") |
|---|
| 200 | 185 | |
|---|
| 186 | +static irqreturn_t irq_handler(void *private) |
|---|
| 187 | +{ |
|---|
| 188 | + struct eventfd_ctx *ev_ctx = private; |
|---|
| 189 | + |
|---|
| 190 | + eventfd_signal(ev_ctx, 1); |
|---|
| 191 | + return IRQ_HANDLED; |
|---|
| 192 | +} |
|---|
| 193 | + |
|---|
| 194 | +static void irq_free(void *private) |
|---|
| 195 | +{ |
|---|
| 196 | + struct eventfd_ctx *ev_ctx = private; |
|---|
| 197 | + |
|---|
| 198 | + eventfd_ctx_put(ev_ctx); |
|---|
| 199 | +} |
|---|
| 200 | + |
|---|
| 201 | 201 | static long afu_ioctl(struct file *file, unsigned int cmd, |
|---|
| 202 | 202 | unsigned long args) |
|---|
| 203 | 203 | { |
|---|
| 204 | 204 | struct ocxl_context *ctx = file->private_data; |
|---|
| 205 | 205 | struct ocxl_ioctl_irq_fd irq_fd; |
|---|
| 206 | + struct eventfd_ctx *ev_ctx; |
|---|
| 207 | + int irq_id; |
|---|
| 206 | 208 | u64 irq_offset; |
|---|
| 207 | 209 | long rc; |
|---|
| 210 | + bool closed; |
|---|
| 208 | 211 | |
|---|
| 209 | 212 | pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid, |
|---|
| 210 | 213 | CMD_STR(cmd)); |
|---|
| 211 | 214 | |
|---|
| 212 | | - if (ctx->status == CLOSED) |
|---|
| 215 | + mutex_lock(&ctx->status_mutex); |
|---|
| 216 | + closed = (ctx->status == CLOSED); |
|---|
| 217 | + mutex_unlock(&ctx->status_mutex); |
|---|
| 218 | + |
|---|
| 219 | + if (closed) |
|---|
| 213 | 220 | return -EIO; |
|---|
| 214 | 221 | |
|---|
| 215 | 222 | switch (cmd) { |
|---|
| .. | .. |
|---|
| 219 | 226 | break; |
|---|
| 220 | 227 | |
|---|
| 221 | 228 | case OCXL_IOCTL_IRQ_ALLOC: |
|---|
| 222 | | - rc = ocxl_afu_irq_alloc(ctx, &irq_offset); |
|---|
| 229 | + rc = ocxl_afu_irq_alloc(ctx, &irq_id); |
|---|
| 223 | 230 | if (!rc) { |
|---|
| 231 | + irq_offset = ocxl_irq_id_to_offset(ctx, irq_id); |
|---|
| 224 | 232 | rc = copy_to_user((u64 __user *) args, &irq_offset, |
|---|
| 225 | 233 | sizeof(irq_offset)); |
|---|
| 226 | 234 | if (rc) { |
|---|
| 227 | | - ocxl_afu_irq_free(ctx, irq_offset); |
|---|
| 235 | + ocxl_afu_irq_free(ctx, irq_id); |
|---|
| 228 | 236 | return -EFAULT; |
|---|
| 229 | 237 | } |
|---|
| 230 | 238 | } |
|---|
| .. | .. |
|---|
| 235 | 243 | sizeof(irq_offset)); |
|---|
| 236 | 244 | if (rc) |
|---|
| 237 | 245 | return -EFAULT; |
|---|
| 238 | | - rc = ocxl_afu_irq_free(ctx, irq_offset); |
|---|
| 246 | + irq_id = ocxl_irq_offset_to_id(ctx, irq_offset); |
|---|
| 247 | + rc = ocxl_afu_irq_free(ctx, irq_id); |
|---|
| 239 | 248 | break; |
|---|
| 240 | 249 | |
|---|
| 241 | 250 | case OCXL_IOCTL_IRQ_SET_FD: |
|---|
| .. | .. |
|---|
| 245 | 254 | return -EFAULT; |
|---|
| 246 | 255 | if (irq_fd.reserved) |
|---|
| 247 | 256 | return -EINVAL; |
|---|
| 248 | | - rc = ocxl_afu_irq_set_fd(ctx, irq_fd.irq_offset, |
|---|
| 249 | | - irq_fd.eventfd); |
|---|
| 257 | + irq_id = ocxl_irq_offset_to_id(ctx, irq_fd.irq_offset); |
|---|
| 258 | + ev_ctx = eventfd_ctx_fdget(irq_fd.eventfd); |
|---|
| 259 | + if (IS_ERR(ev_ctx)) |
|---|
| 260 | + return PTR_ERR(ev_ctx); |
|---|
| 261 | + rc = ocxl_irq_set_handler(ctx, irq_id, irq_handler, irq_free, ev_ctx); |
|---|
| 262 | + if (rc) |
|---|
| 263 | + eventfd_ctx_put(ev_ctx); |
|---|
| 250 | 264 | break; |
|---|
| 251 | 265 | |
|---|
| 252 | 266 | case OCXL_IOCTL_GET_METADATA: |
|---|
| .. | .. |
|---|
| 469 | 483 | .release = afu_release, |
|---|
| 470 | 484 | }; |
|---|
| 471 | 485 | |
|---|
| 472 | | -int ocxl_create_cdev(struct ocxl_afu *afu) |
|---|
| 486 | +// Free the info struct |
|---|
| 487 | +static void info_release(struct device *dev) |
|---|
| 488 | +{ |
|---|
| 489 | + struct ocxl_file_info *info = container_of(dev, struct ocxl_file_info, dev); |
|---|
| 490 | + |
|---|
| 491 | + ocxl_afu_put(info->afu); |
|---|
| 492 | + kfree(info); |
|---|
| 493 | +} |
|---|
| 494 | + |
|---|
| 495 | +static int ocxl_file_make_visible(struct ocxl_file_info *info) |
|---|
| 473 | 496 | { |
|---|
| 474 | 497 | int rc; |
|---|
| 475 | 498 | |
|---|
| 476 | | - cdev_init(&afu->cdev, &ocxl_afu_fops); |
|---|
| 477 | | - rc = cdev_add(&afu->cdev, afu->dev.devt, 1); |
|---|
| 499 | + cdev_init(&info->cdev, &ocxl_afu_fops); |
|---|
| 500 | + rc = cdev_add(&info->cdev, info->dev.devt, 1); |
|---|
| 478 | 501 | if (rc) { |
|---|
| 479 | | - dev_err(&afu->dev, "Unable to add afu char device: %d\n", rc); |
|---|
| 502 | + dev_err(&info->dev, "Unable to add afu char device: %d\n", rc); |
|---|
| 480 | 503 | return rc; |
|---|
| 481 | 504 | } |
|---|
| 505 | + |
|---|
| 482 | 506 | return 0; |
|---|
| 483 | 507 | } |
|---|
| 484 | 508 | |
|---|
| 485 | | -void ocxl_destroy_cdev(struct ocxl_afu *afu) |
|---|
| 509 | +static void ocxl_file_make_invisible(struct ocxl_file_info *info) |
|---|
| 486 | 510 | { |
|---|
| 487 | | - cdev_del(&afu->cdev); |
|---|
| 511 | + cdev_del(&info->cdev); |
|---|
| 488 | 512 | } |
|---|
| 489 | 513 | |
|---|
| 490 | | -int ocxl_register_afu(struct ocxl_afu *afu) |
|---|
| 514 | +int ocxl_file_register_afu(struct ocxl_afu *afu) |
|---|
| 491 | 515 | { |
|---|
| 492 | 516 | int minor; |
|---|
| 517 | + int rc; |
|---|
| 518 | + struct ocxl_file_info *info; |
|---|
| 519 | + struct ocxl_fn *fn = afu->fn; |
|---|
| 520 | + struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent); |
|---|
| 493 | 521 | |
|---|
| 494 | | - minor = allocate_afu_minor(afu); |
|---|
| 495 | | - if (minor < 0) |
|---|
| 522 | + info = kzalloc(sizeof(*info), GFP_KERNEL); |
|---|
| 523 | + if (info == NULL) |
|---|
| 524 | + return -ENOMEM; |
|---|
| 525 | + |
|---|
| 526 | + minor = allocate_minor(info); |
|---|
| 527 | + if (minor < 0) { |
|---|
| 528 | + kfree(info); |
|---|
| 496 | 529 | return minor; |
|---|
| 497 | | - afu->dev.devt = MKDEV(MAJOR(ocxl_dev), minor); |
|---|
| 498 | | - afu->dev.class = ocxl_class; |
|---|
| 499 | | - return device_register(&afu->dev); |
|---|
| 530 | + } |
|---|
| 531 | + |
|---|
| 532 | + info->dev.parent = &fn->dev; |
|---|
| 533 | + info->dev.devt = MKDEV(MAJOR(ocxl_dev), minor); |
|---|
| 534 | + info->dev.class = ocxl_class; |
|---|
| 535 | + info->dev.release = info_release; |
|---|
| 536 | + |
|---|
| 537 | + info->afu = afu; |
|---|
| 538 | + ocxl_afu_get(afu); |
|---|
| 539 | + |
|---|
| 540 | + rc = dev_set_name(&info->dev, "%s.%s.%hhu", |
|---|
| 541 | + afu->config.name, dev_name(&pci_dev->dev), afu->config.idx); |
|---|
| 542 | + if (rc) |
|---|
| 543 | + goto err_put; |
|---|
| 544 | + |
|---|
| 545 | + rc = device_register(&info->dev); |
|---|
| 546 | + if (rc) |
|---|
| 547 | + goto err_put; |
|---|
| 548 | + |
|---|
| 549 | + rc = ocxl_sysfs_register_afu(info); |
|---|
| 550 | + if (rc) |
|---|
| 551 | + goto err_unregister; |
|---|
| 552 | + |
|---|
| 553 | + rc = ocxl_file_make_visible(info); |
|---|
| 554 | + if (rc) |
|---|
| 555 | + goto err_unregister; |
|---|
| 556 | + |
|---|
| 557 | + ocxl_afu_set_private(afu, info); |
|---|
| 558 | + |
|---|
| 559 | + return 0; |
|---|
| 560 | + |
|---|
| 561 | +err_unregister: |
|---|
| 562 | + ocxl_sysfs_unregister_afu(info); // safe to call even if register failed |
|---|
| 563 | + free_minor(info); |
|---|
| 564 | + device_unregister(&info->dev); |
|---|
| 565 | + return rc; |
|---|
| 566 | +err_put: |
|---|
| 567 | + ocxl_afu_put(afu); |
|---|
| 568 | + free_minor(info); |
|---|
| 569 | + kfree(info); |
|---|
| 570 | + return rc; |
|---|
| 500 | 571 | } |
|---|
| 501 | 572 | |
|---|
| 502 | | -void ocxl_unregister_afu(struct ocxl_afu *afu) |
|---|
| 573 | +void ocxl_file_unregister_afu(struct ocxl_afu *afu) |
|---|
| 503 | 574 | { |
|---|
| 504 | | - free_afu_minor(afu); |
|---|
| 575 | + struct ocxl_file_info *info = ocxl_afu_get_private(afu); |
|---|
| 576 | + |
|---|
| 577 | + if (!info) |
|---|
| 578 | + return; |
|---|
| 579 | + |
|---|
| 580 | + ocxl_file_make_invisible(info); |
|---|
| 581 | + ocxl_sysfs_unregister_afu(info); |
|---|
| 582 | + free_minor(info); |
|---|
| 583 | + device_unregister(&info->dev); |
|---|
| 505 | 584 | } |
|---|
| 506 | 585 | |
|---|
| 507 | 586 | static char *ocxl_devnode(struct device *dev, umode_t *mode) |
|---|