| .. | .. |
|---|
| 31 | 31 | #include "kfd_priv.h" |
|---|
| 32 | 32 | #include "kfd_mqd_manager.h" |
|---|
| 33 | 33 | |
|---|
| 34 | | -#define KFD_UNMAP_LATENCY_MS (4000) |
|---|
| 35 | | -#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000) |
|---|
| 36 | | -#define KFD_SDMA_QUEUES_PER_ENGINE (2) |
|---|
| 34 | + |
|---|
| 35 | +#define VMID_NUM 16 |
|---|
| 37 | 36 | |
|---|
| 38 | 37 | struct device_process_node { |
|---|
| 39 | 38 | struct qcm_process_device *qpd; |
|---|
| .. | .. |
|---|
| 48 | 47 | * @destroy_queue: Queue destruction routine. |
|---|
| 49 | 48 | * |
|---|
| 50 | 49 | * @update_queue: Queue update routine. |
|---|
| 51 | | - * |
|---|
| 52 | | - * @get_mqd_manager: Returns the mqd manager according to the mqd type. |
|---|
| 53 | 50 | * |
|---|
| 54 | 51 | * @exeute_queues: Dispatches the queues list to the H/W. |
|---|
| 55 | 52 | * |
|---|
| .. | .. |
|---|
| 82 | 79 | * |
|---|
| 83 | 80 | * @restore_process_queues: Restore all evicted queues queues of a process |
|---|
| 84 | 81 | * |
|---|
| 82 | + * @get_wave_state: Retrieves context save state and optionally copies the |
|---|
| 83 | + * control stack, if kept in the MQD, to the given userspace address. |
|---|
| 85 | 84 | */ |
|---|
| 86 | 85 | |
|---|
| 87 | 86 | struct device_queue_manager_ops { |
|---|
| .. | .. |
|---|
| 96 | 95 | int (*update_queue)(struct device_queue_manager *dqm, |
|---|
| 97 | 96 | struct queue *q); |
|---|
| 98 | 97 | |
|---|
| 99 | | - struct mqd_manager * (*get_mqd_manager) |
|---|
| 100 | | - (struct device_queue_manager *dqm, |
|---|
| 101 | | - enum KFD_MQD_TYPE type); |
|---|
| 102 | | - |
|---|
| 103 | 98 | int (*register_process)(struct device_queue_manager *dqm, |
|---|
| 104 | 99 | struct qcm_process_device *qpd); |
|---|
| 105 | 100 | |
|---|
| .. | .. |
|---|
| 109 | 104 | int (*initialize)(struct device_queue_manager *dqm); |
|---|
| 110 | 105 | int (*start)(struct device_queue_manager *dqm); |
|---|
| 111 | 106 | int (*stop)(struct device_queue_manager *dqm); |
|---|
| 107 | + void (*pre_reset)(struct device_queue_manager *dqm); |
|---|
| 112 | 108 | void (*uninitialize)(struct device_queue_manager *dqm); |
|---|
| 113 | 109 | int (*create_kernel_queue)(struct device_queue_manager *dqm, |
|---|
| 114 | 110 | struct kernel_queue *kq, |
|---|
| .. | .. |
|---|
| 137 | 133 | struct qcm_process_device *qpd); |
|---|
| 138 | 134 | int (*restore_process_queues)(struct device_queue_manager *dqm, |
|---|
| 139 | 135 | struct qcm_process_device *qpd); |
|---|
| 136 | + |
|---|
| 137 | + int (*get_wave_state)(struct device_queue_manager *dqm, |
|---|
| 138 | + struct queue *q, |
|---|
| 139 | + void __user *ctl_stack, |
|---|
| 140 | + u32 *ctl_stack_used_size, |
|---|
| 141 | + u32 *save_area_used_size); |
|---|
| 140 | 142 | }; |
|---|
| 141 | 143 | |
|---|
| 142 | 144 | struct device_queue_manager_asic_ops { |
|---|
| .. | .. |
|---|
| 151 | 153 | void (*init_sdma_vm)(struct device_queue_manager *dqm, |
|---|
| 152 | 154 | struct queue *q, |
|---|
| 153 | 155 | struct qcm_process_device *qpd); |
|---|
| 156 | + struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type, |
|---|
| 157 | + struct kfd_dev *dev); |
|---|
| 154 | 158 | }; |
|---|
| 155 | 159 | |
|---|
| 156 | 160 | /** |
|---|
| .. | .. |
|---|
| 176 | 180 | struct list_head queues; |
|---|
| 177 | 181 | unsigned int saved_flags; |
|---|
| 178 | 182 | unsigned int processes_count; |
|---|
| 179 | | - unsigned int queue_count; |
|---|
| 180 | | - unsigned int sdma_queue_count; |
|---|
| 183 | + unsigned int active_queue_count; |
|---|
| 184 | + unsigned int active_cp_queue_count; |
|---|
| 185 | + unsigned int gws_queue_count; |
|---|
| 181 | 186 | unsigned int total_queue_count; |
|---|
| 182 | 187 | unsigned int next_pipe_to_allocate; |
|---|
| 183 | 188 | unsigned int *allocated_queues; |
|---|
| 184 | | - unsigned int sdma_bitmap; |
|---|
| 185 | | - unsigned int vmid_bitmap; |
|---|
| 189 | + uint64_t sdma_bitmap; |
|---|
| 190 | + uint64_t xgmi_sdma_bitmap; |
|---|
| 191 | + /* the pasid mapping for each kfd vmid */ |
|---|
| 192 | + uint16_t vmid_pasid[VMID_NUM]; |
|---|
| 186 | 193 | uint64_t pipelines_addr; |
|---|
| 187 | | - struct kfd_mem_obj *pipeline_mem; |
|---|
| 188 | 194 | uint64_t fence_gpu_addr; |
|---|
| 189 | | - unsigned int *fence_addr; |
|---|
| 195 | + uint64_t *fence_addr; |
|---|
| 190 | 196 | struct kfd_mem_obj *fence_mem; |
|---|
| 191 | 197 | bool active_runlist; |
|---|
| 192 | 198 | int sched_policy; |
|---|
| 193 | 199 | |
|---|
| 194 | 200 | /* hw exception */ |
|---|
| 195 | 201 | bool is_hws_hang; |
|---|
| 202 | + bool is_resetting; |
|---|
| 196 | 203 | struct work_struct hw_exception_work; |
|---|
| 204 | + struct kfd_mem_obj hiq_sdma_mqd; |
|---|
| 205 | + bool sched_running; |
|---|
| 197 | 206 | }; |
|---|
| 198 | 207 | |
|---|
| 199 | 208 | void device_queue_manager_init_cik( |
|---|
| .. | .. |
|---|
| 206 | 215 | struct device_queue_manager_asic_ops *asic_ops); |
|---|
| 207 | 216 | void device_queue_manager_init_v9( |
|---|
| 208 | 217 | struct device_queue_manager_asic_ops *asic_ops); |
|---|
| 218 | +void device_queue_manager_init_v10_navi10( |
|---|
| 219 | + struct device_queue_manager_asic_ops *asic_ops); |
|---|
| 209 | 220 | void program_sh_mem_settings(struct device_queue_manager *dqm, |
|---|
| 210 | 221 | struct qcm_process_device *qpd); |
|---|
| 211 | | -unsigned int get_queues_num(struct device_queue_manager *dqm); |
|---|
| 222 | +unsigned int get_cp_queues_num(struct device_queue_manager *dqm); |
|---|
| 212 | 223 | unsigned int get_queues_per_pipe(struct device_queue_manager *dqm); |
|---|
| 213 | 224 | unsigned int get_pipes_per_mec(struct device_queue_manager *dqm); |
|---|
| 214 | 225 | unsigned int get_num_sdma_queues(struct device_queue_manager *dqm); |
|---|
| 226 | +unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm); |
|---|
| 215 | 227 | |
|---|
| 216 | 228 | static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) |
|---|
| 217 | 229 | { |
|---|
| .. | .. |
|---|
| 231 | 243 | static inline void dqm_lock(struct device_queue_manager *dqm) |
|---|
| 232 | 244 | { |
|---|
| 233 | 245 | mutex_lock(&dqm->lock_hidden); |
|---|
| 234 | | - dqm->saved_flags = memalloc_nofs_save(); |
|---|
| 246 | + dqm->saved_flags = memalloc_noreclaim_save(); |
|---|
| 235 | 247 | } |
|---|
| 236 | 248 | static inline void dqm_unlock(struct device_queue_manager *dqm) |
|---|
| 237 | 249 | { |
|---|
| 238 | | - memalloc_nofs_restore(dqm->saved_flags); |
|---|
| 250 | + memalloc_noreclaim_restore(dqm->saved_flags); |
|---|
| 239 | 251 | mutex_unlock(&dqm->lock_hidden); |
|---|
| 240 | 252 | } |
|---|
| 241 | 253 | |
|---|
| 254 | +static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val) |
|---|
| 255 | +{ |
|---|
| 256 | + /* |
|---|
| 257 | + * SDMA activity counter is stored at queue's RPTR + 0x8 location. |
|---|
| 258 | + */ |
|---|
| 259 | + return get_user(*val, q_rptr + 1); |
|---|
| 260 | +} |
|---|
| 242 | 261 | #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ |
|---|