| .. | .. |
|---|
| 124 | 124 | job->use_core_num = (args->core_mask & RKNPU_CORE0_MASK) + |
|---|
| 125 | 125 | ((args->core_mask & RKNPU_CORE1_MASK) >> 1) + |
|---|
| 126 | 126 | ((args->core_mask & RKNPU_CORE2_MASK) >> 2); |
|---|
| 127 | | - job->run_count = job->use_core_num; |
|---|
| 128 | | - job->interrupt_count = job->use_core_num; |
|---|
| 127 | + atomic_set(&job->run_count, job->use_core_num); |
|---|
| 128 | + atomic_set(&job->interrupt_count, job->use_core_num); |
|---|
| 129 | 129 | #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM |
|---|
| 130 | 130 | task_obj = (struct rknpu_gem_object *)(uintptr_t)args->task_obj_addr; |
|---|
| 131 | 131 | if (task_obj) |
|---|
| .. | .. |
|---|
| 162 | 162 | unsigned long flags; |
|---|
| 163 | 163 | int wait_count = 0; |
|---|
| 164 | 164 | int ret = -EINVAL; |
|---|
| 165 | + int i = 0; |
|---|
| 165 | 166 | |
|---|
| 166 | 167 | subcore_data = &rknpu_dev->subcore_datas[core_index]; |
|---|
| 167 | 168 | |
|---|
| 168 | 169 | do { |
|---|
| 169 | | - ret = wait_event_interruptible_timeout( |
|---|
| 170 | | - subcore_data->job_done_wq, |
|---|
| 171 | | - job->flags & RKNPU_JOB_DONE || rknpu_dev->soft_reseting, |
|---|
| 172 | | - msecs_to_jiffies(args->timeout)); |
|---|
| 170 | + ret = wait_event_timeout(subcore_data->job_done_wq, |
|---|
| 171 | + job->flags & RKNPU_JOB_DONE || |
|---|
| 172 | + rknpu_dev->soft_reseting, |
|---|
| 173 | + msecs_to_jiffies(args->timeout)); |
|---|
| 173 | 174 | if (++wait_count >= 3) |
|---|
| 174 | 175 | break; |
|---|
| 175 | 176 | } while (ret == 0 && job->in_queue[core_index]); |
|---|
| 176 | 177 | |
|---|
| 177 | 178 | if (job->in_queue[core_index]) { |
|---|
| 178 | 179 | spin_lock_irqsave(&rknpu_dev->lock, flags); |
|---|
| 179 | | - list_del_init(&job->head[core_index]); |
|---|
| 180 | 180 | subcore_data->task_num -= rknn_get_task_number(job, core_index); |
|---|
| 181 | | - job->in_queue[core_index] = false; |
|---|
| 181 | + if (job->use_core_num == 1) { |
|---|
| 182 | + list_del_init(&job->head[core_index]); |
|---|
| 183 | + job->in_queue[core_index] = false; |
|---|
| 184 | + } else if (job->use_core_num > 1) { |
|---|
| 185 | + for (i = 0; i < job->use_core_num; i++) { |
|---|
| 186 | + if (job->in_queue[i]) { |
|---|
| 187 | + list_del_init(&job->head[i]); |
|---|
| 188 | + job->in_queue[i] = false; |
|---|
| 189 | + } |
|---|
| 190 | + } |
|---|
| 191 | + } |
|---|
| 182 | 192 | spin_unlock_irqrestore(&rknpu_dev->lock, flags); |
|---|
| 183 | 193 | return ret < 0 ? ret : -EINVAL; |
|---|
| 184 | 194 | } |
|---|
| .. | .. |
|---|
| 193 | 203 | args->task_counter = 0; |
|---|
| 194 | 204 | rknpu_core_base = rknpu_dev->base[core_index]; |
|---|
| 195 | 205 | if (args->flags & RKNPU_JOB_PC) { |
|---|
| 196 | | - uint32_t task_status = |
|---|
| 197 | | - REG_READ(RKNPU_OFFSET_PC_TASK_STATUS); |
|---|
| 206 | + uint32_t task_status = REG_READ( |
|---|
| 207 | + rknpu_dev->config->pc_task_status_offset); |
|---|
| 198 | 208 | args->task_counter = |
|---|
| 199 | 209 | (task_status & |
|---|
| 200 | 210 | rknpu_dev->config->pc_task_number_mask); |
|---|
| .. | .. |
|---|
| 348 | 358 | list_del_init(&job->head[core_index]); |
|---|
| 349 | 359 | job->in_queue[core_index] = false; |
|---|
| 350 | 360 | subcore_data->job = job; |
|---|
| 351 | | - job->run_count--; |
|---|
| 352 | 361 | job->hw_recoder_time = ktime_get(); |
|---|
| 353 | 362 | spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); |
|---|
| 354 | 363 | |
|---|
| 355 | | - if (job->run_count == 0) { |
|---|
| 364 | + if (atomic_dec_and_test(&job->run_count)) { |
|---|
| 356 | 365 | if (job->args->core_mask & RKNPU_CORE0_MASK) |
|---|
| 357 | 366 | job->ret = rknpu_job_commit(job, 0); |
|---|
| 358 | 367 | if (job->args->core_mask & RKNPU_CORE1_MASK) |
|---|
| .. | .. |
|---|
| 374 | 383 | spin_lock_irqsave(&rknpu_dev->irq_lock, flags); |
|---|
| 375 | 384 | subcore_data->job = NULL; |
|---|
| 376 | 385 | subcore_data->task_num -= rknn_get_task_number(job, core_index); |
|---|
| 377 | | - job->interrupt_count--; |
|---|
| 378 | 386 | subcore_data->timer.busy_time += |
|---|
| 379 | 387 | ktime_us_delta(now, job->hw_recoder_time); |
|---|
| 380 | 388 | spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); |
|---|
| 381 | 389 | |
|---|
| 382 | | - if (job->interrupt_count == 0) { |
|---|
| 390 | + if (atomic_dec_and_test(&job->interrupt_count)) { |
|---|
| 383 | 391 | int use_core_num = job->use_core_num; |
|---|
| 384 | 392 | |
|---|
| 385 | 393 | job->flags |= RKNPU_JOB_DONE; |
|---|
| .. | .. |
|---|
| 439 | 447 | |
|---|
| 440 | 448 | job->args->core_mask = rknpu_core_mask(core_index); |
|---|
| 441 | 449 | job->use_core_num = 1; |
|---|
| 442 | | - job->interrupt_count = 1; |
|---|
| 443 | | - job->run_count = 1; |
|---|
| 450 | + atomic_set(&job->run_count, job->use_core_num); |
|---|
| 451 | + atomic_set(&job->interrupt_count, job->use_core_num); |
|---|
| 444 | 452 | } |
|---|
| 445 | 453 | |
|---|
| 454 | + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); |
|---|
| 446 | 455 | for (i = 0; i < rknpu_dev->config->num_irqs; i++) { |
|---|
| 447 | 456 | if (job->args->core_mask & rknpu_core_mask(i)) { |
|---|
| 448 | 457 | subcore_data = &rknpu_dev->subcore_datas[i]; |
|---|
| 449 | | - spin_lock_irqsave(&rknpu_dev->irq_lock, flags); |
|---|
| 450 | 458 | list_add_tail(&job->head[i], &subcore_data->todo_list); |
|---|
| 451 | 459 | subcore_data->task_num += rknn_get_task_number(job, i); |
|---|
| 452 | 460 | job->in_queue[i] = true; |
|---|
| 453 | | - spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); |
|---|
| 454 | 461 | } |
|---|
| 455 | 462 | } |
|---|
| 463 | + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); |
|---|
| 456 | 464 | |
|---|
| 457 | 465 | for (i = 0; i < rknpu_dev->config->num_irqs; i++) { |
|---|
| 458 | 466 | if (job->args->core_mask & rknpu_core_mask(i)) |
|---|
| .. | .. |
|---|
| 464 | 472 | { |
|---|
| 465 | 473 | struct rknpu_device *rknpu_dev = job->rknpu_dev; |
|---|
| 466 | 474 | struct rknpu_subcore_data *subcore_data = NULL; |
|---|
| 467 | | - int core_index = rknpu_core_index(job->args->core_mask); |
|---|
| 468 | | - void __iomem *rknpu_core_base = rknpu_dev->base[core_index]; |
|---|
| 469 | 475 | unsigned long flags; |
|---|
| 470 | 476 | int i = 0; |
|---|
| 471 | 477 | |
|---|
| .. | .. |
|---|
| 485 | 491 | } |
|---|
| 486 | 492 | |
|---|
| 487 | 493 | if (job->ret == -ETIMEDOUT) { |
|---|
| 488 | | - LOG_ERROR( |
|---|
| 489 | | - "job timeout, flags: %#x, irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x, elapsed time: %lldus\n", |
|---|
| 490 | | - job->flags, REG_READ(RKNPU_OFFSET_INT_STATUS), |
|---|
| 491 | | - REG_READ(RKNPU_OFFSET_INT_RAW_STATUS), |
|---|
| 492 | | - job->int_mask[core_index], |
|---|
| 493 | | - (REG_READ(RKNPU_OFFSET_PC_TASK_STATUS) & |
|---|
| 494 | | - rknpu_dev->config->pc_task_number_mask), |
|---|
| 495 | | - ktime_to_us(ktime_sub(ktime_get(), job->timestamp))); |
|---|
| 494 | + LOG_ERROR("job timeout, flags: %#x:\n", job->flags); |
|---|
| 495 | + for (i = 0; i < rknpu_dev->config->num_irqs; i++) { |
|---|
| 496 | + if (job->args->core_mask & rknpu_core_mask(i)) { |
|---|
| 497 | + void __iomem *rknpu_core_base = |
|---|
| 498 | + rknpu_dev->base[i]; |
|---|
| 499 | + LOG_ERROR( |
|---|
| 500 | + "\tcore %d irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x, elapsed time: %lldus\n", |
|---|
| 501 | + i, REG_READ(RKNPU_OFFSET_INT_STATUS), |
|---|
| 502 | + REG_READ(RKNPU_OFFSET_INT_RAW_STATUS), |
|---|
| 503 | + job->int_mask[i], |
|---|
| 504 | + (REG_READ( |
|---|
| 505 | + rknpu_dev->config |
|---|
| 506 | + ->pc_task_status_offset) & |
|---|
| 507 | + rknpu_dev->config->pc_task_number_mask), |
|---|
| 508 | + ktime_to_us(ktime_sub(ktime_get(), |
|---|
| 509 | + job->timestamp))); |
|---|
| 510 | + } |
|---|
| 511 | + } |
|---|
| 496 | 512 | rknpu_soft_reset(rknpu_dev); |
|---|
| 497 | 513 | } else { |
|---|
| 498 | 514 | LOG_ERROR( |
|---|
| .. | .. |
|---|
| 560 | 576 | "invalid irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x\n", |
|---|
| 561 | 577 | status, REG_READ(RKNPU_OFFSET_INT_RAW_STATUS), |
|---|
| 562 | 578 | job->int_mask[core_index], |
|---|
| 563 | | - (REG_READ(RKNPU_OFFSET_PC_TASK_STATUS) & |
|---|
| 579 | + (REG_READ(rknpu_dev->config->pc_task_status_offset) & |
|---|
| 564 | 580 | rknpu_dev->config->pc_task_number_mask)); |
|---|
| 565 | 581 | REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR); |
|---|
| 566 | 582 | return IRQ_HANDLED; |
|---|
| .. | .. |
|---|
| 779 | 795 | return -EINVAL; |
|---|
| 780 | 796 | |
|---|
| 781 | 797 | *version = REG_READ(RKNPU_OFFSET_VERSION) + |
|---|
| 782 | | - REG_READ(RKNPU_OFFSET_VERSION_NUM); |
|---|
| 798 | + (REG_READ(RKNPU_OFFSET_VERSION_NUM) & 0xffff); |
|---|
| 783 | 799 | |
|---|
| 784 | 800 | return 0; |
|---|
| 785 | 801 | } |
|---|
| .. | .. |
|---|
| 853 | 869 | |
|---|
| 854 | 870 | spin_lock(&rknpu_dev->lock); |
|---|
| 855 | 871 | |
|---|
| 856 | | - REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); |
|---|
| 857 | | - REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); |
|---|
| 872 | + if (rknpu_dev->config->pc_dma_ctrl) { |
|---|
| 873 | + uint32_t pc_data_addr = REG_READ(RKNPU_OFFSET_PC_DATA_ADDR); |
|---|
| 874 | + |
|---|
| 875 | + REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR); |
|---|
| 876 | + REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); |
|---|
| 877 | + REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); |
|---|
| 878 | + REG_WRITE(pc_data_addr, RKNPU_OFFSET_PC_DATA_ADDR); |
|---|
| 879 | + } else { |
|---|
| 880 | + REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); |
|---|
| 881 | + REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); |
|---|
| 882 | + } |
|---|
| 858 | 883 | |
|---|
| 859 | 884 | spin_unlock(&rknpu_dev->lock); |
|---|
| 860 | 885 | |
|---|