/* * * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms * of such GNU licence. * * A copy of the licence is included with the program, and can also be obtained * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301, USA. * */ /* * Register-based HW access backend specific APIs */ #include #include #include #include #include #include #include #include #include #include #include /* Return whether the specified ringbuffer is empty. HW access lock must be * held */ #define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) /* Return number of atoms currently in the specified ringbuffer. HW access lock * must be held */ #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) static void kbase_gpu_release_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom, ktime_t *end_timestamp); /** * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer * @kbdev: Device pointer * @katom: Atom to enqueue * * Context: Caller must hold the HW access lock */ static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom) { struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr]; WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE); lockdep_assert_held(&kbdev->hwaccess_lock); rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom; rb->write_idx++; katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; } /** * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once * it has been completed * @kbdev: Device pointer * @js: Job slot to remove atom from * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in * which case current time will be used. * * Context: Caller must hold the HW access lock * * Return: Atom removed from ringbuffer */ static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev, int js, ktime_t *end_timestamp) { struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; struct kbase_jd_atom *katom; if (SLOT_RB_EMPTY(rb)) { WARN(1, "GPU ringbuffer unexpectedly empty\n"); return NULL; } lockdep_assert_held(&kbdev->hwaccess_lock); katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom; kbase_gpu_release_atom(kbdev, katom, end_timestamp); rb->read_idx++; katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB; kbase_js_debug_log_current_affinities(kbdev); return katom; } struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js, int idx) { struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; lockdep_assert_held(&kbdev->hwaccess_lock); if ((SLOT_RB_ENTRIES(rb) - 1) < idx) return NULL; /* idx out of range */ return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom; } struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev, int js) { return kbase_gpu_inspect(kbdev, js, 0); } struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev, int js) { struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js]; if (SLOT_RB_EMPTY(rb)) return NULL; return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom; } /** * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently * on the GPU * @kbdev: Device pointer * @js: Job slot to inspect * * Return: true if there are atoms on the GPU for slot js, * false otherwise */ static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js) { int i; lockdep_assert_held(&kbdev->hwaccess_lock); for (i = 0; i < SLOT_RB_SIZE; i++) { struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); if (!katom) return false; if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED || katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY) return true; } return false; } /** * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms * currently on the GPU * @kbdev: Device pointer * * Return: true if there are any atoms on the GPU, false otherwise */ static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev) { int js; int i; lockdep_assert_held(&kbdev->hwaccess_lock); for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { for (i = 0; i < SLOT_RB_SIZE; i++) { struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) return true; } } return false; } int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js) { int nr = 0; int i; lockdep_assert_held(&kbdev->hwaccess_lock); for (i = 0; i < SLOT_RB_SIZE; i++) { struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); if (katom && (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)) nr++; } return nr; } int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js) { int nr = 0; int i; lockdep_assert_held(&kbdev->hwaccess_lock); for (i = 0; i < SLOT_RB_SIZE; i++) { if (kbase_gpu_inspect(kbdev, js, i)) nr++; } return nr; } static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js, enum kbase_atom_gpu_rb_state min_rb_state) { int nr = 0; int i; lockdep_assert_held(&kbdev->hwaccess_lock); for (i = 0; i < SLOT_RB_SIZE; i++) { struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); if (katom && (katom->gpu_rb_state >= min_rb_state)) nr++; } return nr; } /** * check_secure_atom - Check if the given atom is in the given secure state and * has a ringbuffer state of at least * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION * @katom: Atom pointer * @secure: Desired secure state * * Return: true if atom is in the given state, false otherwise */ static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure) { if (katom->gpu_rb_state >= KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION && ((kbase_jd_katom_is_protected(katom) && secure) || (!kbase_jd_katom_is_protected(katom) && !secure))) return true; return false; } /** * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given * secure state in the ringbuffers of at least * state * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE * @kbdev: Device pointer * @secure: Desired secure state * * Return: true if any atoms are in the given state, false otherwise */ static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, bool secure) { int js, i; for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { for (i = 0; i < SLOT_RB_SIZE; i++) { struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i); if (katom) { if (check_secure_atom(katom, secure)) return true; } } } return false; } int kbase_backend_slot_free(struct kbase_device *kbdev, int js) { if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_NOT_PENDING) { /* The GPU is being reset - so prevent submission */ return 0; } return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js); } static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, struct kbase_jd_atom *katom); static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev, int js, struct kbase_jd_atom *katom) { /* The most recently checked affinity. Having this at this scope allows * us to guarantee that we've checked the affinity in this function * call. */ u64 recently_chosen_affinity = 0; bool chosen_affinity = false; bool retry; do { retry = false; /* NOTE: The following uses a number of FALLTHROUGHs to optimize * the calls to this function. Ending of the function is * indicated by BREAK OUT */ switch (katom->coreref_state) { /* State when job is first attempted to be run */ case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: KBASE_DEBUG_ASSERT(katom->affinity == 0); /* Compute affinity */ if (false == kbase_js_choose_affinity( &recently_chosen_affinity, kbdev, katom, js)) { /* No cores are currently available */ /* *** BREAK OUT: No state transition *** */ break; } chosen_affinity = true; /* Request the cores */ kbase_pm_request_cores(kbdev, katom->core_req & BASE_JD_REQ_T, recently_chosen_affinity); katom->affinity = recently_chosen_affinity; /* Proceed to next state */ katom->coreref_state = KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: { enum kbase_pm_cores_ready cores_ready; KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T)); cores_ready = kbase_pm_register_inuse_cores( kbdev, katom->core_req & BASE_JD_REQ_T, katom->affinity); if (cores_ready == KBASE_NEW_AFFINITY) { /* Affinity no longer valid - return to * previous state */ kbasep_js_job_check_deref_cores(kbdev, katom); KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_REGISTER_INUSE_FAILED, katom->kctx, katom, katom->jc, js, (u32) katom->affinity); /* *** BREAK OUT: Return to previous * state, retry *** */ retry = true; break; } if (cores_ready == KBASE_CORES_NOT_READY) { /* Stay in this state and return, to * retry at this state later */ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_REGISTER_INUSE_FAILED, katom->kctx, katom, katom->jc, js, (u32) katom->affinity); /* *** BREAK OUT: No state transition * *** */ break; } /* Proceed to next state */ katom->coreref_state = KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; } /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T)); /* Optimize out choosing the affinity twice in the same * function call */ if (chosen_affinity == false) { /* See if the affinity changed since a previous * call. */ if (false == kbase_js_choose_affinity( &recently_chosen_affinity, kbdev, katom, js)) { /* No cores are currently available */ kbasep_js_job_check_deref_cores(kbdev, katom); KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_REQUEST_ON_RECHECK_FAILED, katom->kctx, katom, katom->jc, js, (u32) recently_chosen_affinity); /* *** BREAK OUT: Transition to lower * state *** */ break; } chosen_affinity = true; } /* Now see if this requires a different set of cores */ if (recently_chosen_affinity != katom->affinity) { enum kbase_pm_cores_ready cores_ready; kbase_pm_request_cores(kbdev, katom->core_req & BASE_JD_REQ_T, recently_chosen_affinity); /* Register new cores whilst we still hold the * old ones, to minimize power transitions */ cores_ready = kbase_pm_register_inuse_cores(kbdev, katom->core_req & BASE_JD_REQ_T, recently_chosen_affinity); kbasep_js_job_check_deref_cores(kbdev, katom); /* Fixup the state that was reduced by * deref_cores: */ katom->coreref_state = KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; katom->affinity = recently_chosen_affinity; if (cores_ready == KBASE_NEW_AFFINITY) { /* Affinity no longer valid - return to * previous state */ katom->coreref_state = KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; kbasep_js_job_check_deref_cores(kbdev, katom); KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_REGISTER_INUSE_FAILED, katom->kctx, katom, katom->jc, js, (u32) katom->affinity); /* *** BREAK OUT: Return to previous * state, retry *** */ retry = true; break; } /* Now might be waiting for powerup again, with * a new affinity */ if (cores_ready == KBASE_CORES_NOT_READY) { /* Return to previous state */ katom->coreref_state = KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES; KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_REGISTER_ON_RECHECK_FAILED, katom->kctx, katom, katom->jc, js, (u32) katom->affinity); /* *** BREAK OUT: Transition to lower * state *** */ break; } } /* Proceed to next state */ katom->coreref_state = KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS; /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS: KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T)); KBASE_DEBUG_ASSERT(katom->affinity == recently_chosen_affinity); /* Note: this is where the caller must've taken the * hwaccess_lock */ /* Check for affinity violations - if there are any, * then we just ask the caller to requeue and try again * later */ if (kbase_js_affinity_would_violate(kbdev, js, katom->affinity) != false) { /* Return to previous state */ katom->coreref_state = KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY; /* *** BREAK OUT: Transition to lower state *** */ KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_CORE_REF_AFFINITY_WOULD_VIOLATE, katom->kctx, katom, katom->jc, js, (u32) katom->affinity); break; } /* No affinity violations would result, so the cores are * ready */ katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY; /* *** BREAK OUT: Cores Ready *** */ break; default: KBASE_DEBUG_ASSERT_MSG(false, "Unhandled kbase_atom_coreref_state %d", katom->coreref_state); break; } } while (retry != false); return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY); } static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev, struct kbase_jd_atom *katom) { KBASE_DEBUG_ASSERT(kbdev != NULL); KBASE_DEBUG_ASSERT(katom != NULL); switch (katom->coreref_state) { case KBASE_ATOM_COREREF_STATE_READY: /* State where atom was submitted to the HW - just proceed to * power-down */ KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T)); /* fallthrough */ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: /* State where cores were registered */ KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T)); kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T, katom->affinity); break; case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: /* State where cores were requested, but not registered */ KBASE_DEBUG_ASSERT(katom->affinity != 0 || (katom->core_req & BASE_JD_REQ_T)); kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T, katom->affinity); break; case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: /* Initial state - nothing required */ KBASE_DEBUG_ASSERT(katom->affinity == 0); break; default: KBASE_DEBUG_ASSERT_MSG(false, "Unhandled coreref_state: %d", katom->coreref_state); break; } katom->affinity = 0; katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; } static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev, base_jd_core_req core_req, u64 affinity, enum kbase_atom_coreref_state coreref_state) { KBASE_DEBUG_ASSERT(kbdev != NULL); switch (coreref_state) { case KBASE_ATOM_COREREF_STATE_READY: /* State where atom was submitted to the HW - just proceed to * power-down */ KBASE_DEBUG_ASSERT(affinity != 0 || (core_req & BASE_JD_REQ_T)); /* fallthrough */ case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY: /* State where cores were registered */ KBASE_DEBUG_ASSERT(affinity != 0 || (core_req & BASE_JD_REQ_T)); kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T, affinity); break; case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES: /* State where cores were requested, but not registered */ KBASE_DEBUG_ASSERT(affinity != 0 || (core_req & BASE_JD_REQ_T)); kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T, affinity); break; case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED: /* Initial state - nothing required */ KBASE_DEBUG_ASSERT(affinity == 0); break; default: KBASE_DEBUG_ASSERT_MSG(false, "Unhandled coreref_state: %d", coreref_state); break; } } static void kbase_gpu_release_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom, ktime_t *end_timestamp) { struct kbase_context *kctx = katom->kctx; switch (katom->gpu_rb_state) { case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: /* Should be impossible */ WARN(1, "Attempting to release atom not in ringbuffer\n"); break; case KBASE_ATOM_GPU_RB_SUBMITTED: /* Inform power management at start/finish of atom so it can * update its GPU utilisation metrics. Mark atom as not * submitted beforehand. */ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; kbase_pm_metrics_update(kbdev, end_timestamp); if (katom->core_req & BASE_JD_REQ_PERMON) kbase_pm_release_gpu_cycle_counter_nolock(kbdev); /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, &kbdev->gpu_props.props.raw_props.js_features [katom->slot_nr]); KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]); KBASE_TLSTREAM_TL_NRET_CTX_LPU(kctx, &kbdev->gpu_props.props.raw_props.js_features [katom->slot_nr]); /* fallthrough */ case KBASE_ATOM_GPU_RB_READY: /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr, katom->affinity); /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: break; case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: if (katom->protected_state.enter != KBASE_ATOM_ENTER_PROTECTED_CHECK || katom->protected_state.exit != KBASE_ATOM_EXIT_PROTECTED_CHECK) kbdev->protected_mode_transition = false; if (kbase_jd_katom_is_protected(katom) && (katom->protected_state.enter == KBASE_ATOM_ENTER_PROTECTED_IDLE_L2)) { kbase_vinstr_resume(kbdev->vinstr_ctx); /* Go back to configured model for IPA */ kbase_ipa_model_use_configured_locked(kbdev); } /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_RETURN_TO_JS: break; } katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED; katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; } static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev, struct kbase_jd_atom *katom) { kbase_gpu_release_atom(kbdev, katom, NULL); katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; } static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js) { struct kbase_backend_data *backend = &kbdev->hwaccess.backend; bool slot_busy[3]; if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) return true; slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0, KBASE_ATOM_GPU_RB_WAITING_AFFINITY); slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1, KBASE_ATOM_GPU_RB_WAITING_AFFINITY); slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2, KBASE_ATOM_GPU_RB_WAITING_AFFINITY); if ((js == 2 && !(slot_busy[0] || slot_busy[1])) || (js != 2 && !slot_busy[2])) return true; /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */ if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) || kbase_gpu_atoms_submitted(kbdev, 1) || backend->rmu_workaround_flag)) return false; /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */ if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) || !backend->rmu_workaround_flag)) return false; backend->rmu_workaround_flag = !backend->rmu_workaround_flag; return true; } /** * other_slots_busy - Determine if any job slots other than @js are currently * running atoms * @kbdev: Device pointer * @js: Job slot * * Return: true if any slots other than @js are busy, false otherwise */ static inline bool other_slots_busy(struct kbase_device *kbdev, int js) { int slot; for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) { if (slot == js) continue; if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot, KBASE_ATOM_GPU_RB_SUBMITTED)) return true; } return false; } static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev) { return kbdev->protected_mode; } static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev) { int err = -EINVAL; lockdep_assert_held(&kbdev->hwaccess_lock); WARN_ONCE(!kbdev->protected_ops, "Cannot enter protected mode: protected callbacks not specified.\n"); /* * When entering into protected mode, we must ensure that the * GPU is not operating in coherent mode as well. This is to * ensure that no protected memory can be leaked. */ if (kbdev->system_coherency == COHERENCY_ACE) kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE); if (kbdev->protected_ops) { /* Switch GPU to protected mode */ err = kbdev->protected_ops->protected_mode_enable( kbdev->protected_dev); if (err) dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n", err); else kbdev->protected_mode = true; } return err; } static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev) { lockdep_assert_held(&kbdev->hwaccess_lock); WARN_ONCE(!kbdev->protected_ops, "Cannot exit protected mode: protected callbacks not specified.\n"); if (!kbdev->protected_ops) return -EINVAL; /* The protected mode disable callback will be called as part of reset */ kbase_reset_gpu_silent(kbdev); return 0; } static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbase_jd_atom **katom, int idx, int js) { int err = 0; switch (katom[idx]->protected_state.enter) { case KBASE_ATOM_ENTER_PROTECTED_CHECK: KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev); /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV * should ensure that we are not already transitiong, and that * there are no atoms currently on the GPU. */ WARN_ON(kbdev->protected_mode_transition); WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); kbdev->protected_mode_transition = true; katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_VINSTR; /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_ENTER_PROTECTED_VINSTR: if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) { /* * We can't switch now because * the vinstr core state switch * is not done yet. */ return -EAGAIN; } /* Use generic model for IPA in protected mode */ kbase_ipa_model_use_fallback_locked(kbdev); /* Once reaching this point GPU must be * switched to protected mode or vinstr * re-enabled. */ /* * Not in correct mode, begin protected mode switch. * Entering protected mode requires us to power down the L2, * and drop out of fully coherent mode. */ katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_IDLE_L2; kbase_pm_update_cores_state_nolock(kbdev); /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: /* Avoid unnecessary waiting on non-ACE platforms. */ if (kbdev->current_gpu_coherency_mode == COHERENCY_ACE) { if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { /* * The L2 is still powered, wait for all the users to * finish with it before doing the actual reset. */ return -EAGAIN; } } katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_FINISHED; /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_ENTER_PROTECTED_FINISHED: /* No jobs running, so we can switch GPU mode right now. */ err = kbase_gpu_protected_mode_enter(kbdev); /* * Regardless of result, we are no longer transitioning * the GPU. */ kbdev->protected_mode_transition = false; KBASE_TLSTREAM_AUX_PROTECTED_ENTER_END(kbdev); if (err) { /* * Failed to switch into protected mode, resume * vinstr core and fail atom. */ kbase_vinstr_resume(kbdev->vinstr_ctx); katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); /* Only return if head atom or previous atom * already removed - as atoms must be returned * in order. */ if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { kbase_gpu_dequeue_atom(kbdev, js, NULL); kbase_jm_return_atom_to_js(kbdev, katom[idx]); } /* Go back to configured model for IPA */ kbase_ipa_model_use_configured_locked(kbdev); return -EINVAL; } /* Protected mode sanity checks. */ KBASE_DEBUG_ASSERT_MSG( kbase_jd_katom_is_protected(katom[idx]) == kbase_gpu_in_protected_mode(kbdev), "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", kbase_jd_katom_is_protected(katom[idx]), kbase_gpu_in_protected_mode(kbdev)); katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; } return 0; } static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, struct kbase_jd_atom **katom, int idx, int js) { int err = 0; switch (katom[idx]->protected_state.exit) { case KBASE_ATOM_EXIT_PROTECTED_CHECK: KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev); /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV * should ensure that we are not already transitiong, and that * there are no atoms currently on the GPU. */ WARN_ON(kbdev->protected_mode_transition); WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); /* * Exiting protected mode requires a reset, but first the L2 * needs to be powered down to ensure it's not active when the * reset is issued. */ katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_IDLE_L2; kbdev->protected_mode_transition = true; kbase_pm_update_cores_state_nolock(kbdev); /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) || kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) { /* * The L2 is still powered, wait for all the users to * finish with it before doing the actual reset. */ return -EAGAIN; } katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_RESET; /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_EXIT_PROTECTED_RESET: /* Issue the reset to the GPU */ err = kbase_gpu_protected_mode_reset(kbdev); if (err) { kbdev->protected_mode_transition = false; /* Failed to exit protected mode, fail atom */ katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); /* Only return if head atom or previous atom * already removed - as atoms must be returned * in order */ if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { kbase_gpu_dequeue_atom(kbdev, js, NULL); kbase_jm_return_atom_to_js(kbdev, katom[idx]); } kbase_vinstr_resume(kbdev->vinstr_ctx); /* Use generic model for IPA in protected mode */ kbase_ipa_model_use_fallback_locked(kbdev); return -EINVAL; } katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT; /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: /* A GPU reset is issued when exiting protected mode. Once the * reset is done all atoms' state will also be reset. For this * reason, if the atom is still in this state we can safely * say that the reset has not completed i.e., we have not * finished exiting protected mode yet. */ return -EAGAIN; } return 0; } void kbase_backend_slot_update(struct kbase_device *kbdev) { int js; lockdep_assert_held(&kbdev->hwaccess_lock); for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { struct kbase_jd_atom *katom[2]; int idx; katom[0] = kbase_gpu_inspect(kbdev, js, 0); katom[1] = kbase_gpu_inspect(kbdev, js, 1); WARN_ON(katom[1] && !katom[0]); for (idx = 0; idx < SLOT_RB_SIZE; idx++) { bool cores_ready; int ret; if (!katom[idx]) continue; switch (katom[idx]->gpu_rb_state) { case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: /* Should be impossible */ WARN(1, "Attempting to update atom not in ringbuffer\n"); break; case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: if (katom[idx]->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) break; katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: if (kbase_gpu_check_secure_atoms(kbdev, !kbase_jd_katom_is_protected( katom[idx]))) break; if ((idx == 1) && (kbase_jd_katom_is_protected( katom[0]) != kbase_jd_katom_is_protected( katom[1]))) break; if (kbdev->protected_mode_transition) break; katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION; /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: /* * Exiting protected mode must be done before * the references on the cores are taken as * a power down the L2 is required which * can't happen after the references for this * atom are taken. */ if (!kbase_gpu_in_protected_mode(kbdev) && kbase_jd_katom_is_protected(katom[idx])) { /* Atom needs to transition into protected mode. */ ret = kbase_jm_enter_protected_mode(kbdev, katom, idx, js); if (ret) break; } else if (kbase_gpu_in_protected_mode(kbdev) && !kbase_jd_katom_is_protected(katom[idx])) { /* Atom needs to transition out of protected mode. */ ret = kbase_jm_exit_protected_mode(kbdev, katom, idx, js); if (ret) break; } katom[idx]->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; /* Atom needs no protected mode transition. */ katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE; /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: if (katom[idx]->will_fail_event_code) { kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); /* Set EVENT_DONE so this atom will be completed, not unpulled. */ katom[idx]->event_code = BASE_JD_EVENT_DONE; /* Only return if head atom or previous * atom already removed - as atoms must * be returned in order. */ if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { kbase_gpu_dequeue_atom(kbdev, js, NULL); kbase_jm_return_atom_to_js(kbdev, katom[idx]); } break; } cores_ready = kbasep_js_job_check_ref_cores(kbdev, js, katom[idx]); if (katom[idx]->event_code == BASE_JD_EVENT_PM_EVENT) { katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS; break; } if (!cores_ready) break; kbase_js_affinity_retain_slot_cores(kbdev, js, katom[idx]->affinity); katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_AFFINITY; /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_WAITING_AFFINITY: if (!kbase_gpu_rmu_workaround(kbdev, js)) break; katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_READY: if (idx == 1) { /* Only submit if head atom or previous * atom already submitted */ if ((katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED && katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB)) break; /* If intra-slot serialization in use * then don't submit atom to NEXT slot */ if (kbdev->serialize_jobs & KBASE_SERIALIZE_INTRA_SLOT) break; } /* If inter-slot serialization in use then don't * submit atom if any other slots are in use */ if ((kbdev->serialize_jobs & KBASE_SERIALIZE_INTER_SLOT) && other_slots_busy(kbdev, js)) break; if ((kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) && kbase_reset_gpu_active(kbdev)) break; /* Check if this job needs the cycle counter * enabled before submission */ if (katom[idx]->core_req & BASE_JD_REQ_PERMON) kbase_pm_request_gpu_cycle_counter_l2_is_on( kbdev); kbase_job_hw_submit(kbdev, katom[idx], js); katom[idx]->gpu_rb_state = KBASE_ATOM_GPU_RB_SUBMITTED; /* Inform power management at start/finish of * atom so it can update its GPU utilisation * metrics. */ kbase_pm_metrics_update(kbdev, &katom[idx]->start_timestamp); /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ case KBASE_ATOM_GPU_RB_SUBMITTED: /* Atom submitted to HW, nothing else to do */ break; case KBASE_ATOM_GPU_RB_RETURN_TO_JS: /* Only return if head atom or previous atom * already removed - as atoms must be returned * in order */ if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { kbase_gpu_dequeue_atom(kbdev, js, NULL); kbase_jm_return_atom_to_js(kbdev, katom[idx]); } break; } } } /* Warn if PRLAM-8987 affinity restrictions are violated */ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) || kbase_gpu_atoms_submitted(kbdev, 1)) && kbase_gpu_atoms_submitted(kbdev, 2)); } void kbase_backend_run_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom) { lockdep_assert_held(&kbdev->hwaccess_lock); kbase_gpu_enqueue_atom(kbdev, katom); kbase_backend_slot_update(kbdev); } #define HAS_DEP(katom) (katom->pre_dep || katom->atom_flags & \ (KBASE_KATOM_FLAG_X_DEP_BLOCKED | KBASE_KATOM_FLAG_FAIL_BLOCKER)) bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js) { struct kbase_jd_atom *katom; struct kbase_jd_atom *next_katom; lockdep_assert_held(&kbdev->hwaccess_lock); katom = kbase_gpu_inspect(kbdev, js, 0); next_katom = kbase_gpu_inspect(kbdev, js, 1); if (next_katom && katom->kctx == next_katom->kctx && next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED && HAS_DEP(next_katom) && (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) != 0 || kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) != 0)) { kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), JS_COMMAND_NOP, NULL); next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; KBASE_TLSTREAM_TL_NRET_ATOM_LPU(katom, &kbdev->gpu_props.props.raw_props.js_features [katom->slot_nr]); KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as [katom->kctx->as_nr]); KBASE_TLSTREAM_TL_NRET_CTX_LPU(katom->kctx, &kbdev->gpu_props.props.raw_props.js_features [katom->slot_nr]); return true; } return false; } void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, u32 completion_code, u64 job_tail, ktime_t *end_timestamp) { struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); struct kbase_context *kctx = katom->kctx; lockdep_assert_held(&kbdev->hwaccess_lock); /* * When a hard-stop is followed close after a soft-stop, the completion * code may be set to STOPPED, even though the job is terminated */ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) { if (completion_code == BASE_JD_EVENT_STOPPED && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { completion_code = BASE_JD_EVENT_TERMINATED; } } if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) || (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END)) && completion_code != BASE_JD_EVENT_DONE && !(completion_code & BASE_JD_SW_EVENT)) { /* When a job chain fails, on a T60x or when * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not * flushed. To prevent future evictions causing possible memory * corruption we need to flush the cache manually before any * affected memory gets reused. */ katom->need_cache_flush_cores_retained = katom->affinity; kbase_pm_request_cores(kbdev, false, katom->affinity); } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) { if (kbdev->gpu_props.num_core_groups > 1 && !(katom->affinity & kbdev->gpu_props.props.coherency_info.group[0].core_mask ) && (katom->affinity & kbdev->gpu_props.props.coherency_info.group[1].core_mask )) { dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n"); katom->need_cache_flush_cores_retained = katom->affinity; kbase_pm_request_cores(kbdev, false, katom->affinity); } } katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0); if (completion_code == BASE_JD_EVENT_STOPPED) { struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, 0); /* * Dequeue next atom from ringbuffers on same slot if required. * This atom will already have been removed from the NEXT * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that * the atoms on this slot are returned in the correct order. */ if (next_katom && katom->kctx == next_katom->kctx && next_katom->sched_priority == katom->sched_priority) { kbase_gpu_dequeue_atom(kbdev, js, end_timestamp); kbase_jm_return_atom_to_js(kbdev, next_katom); } } else if (completion_code != BASE_JD_EVENT_DONE) { struct kbasep_js_device_data *js_devdata = &kbdev->js_data; int i; #if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0 KBASE_TRACE_DUMP(kbdev); #endif kbasep_js_clear_submit_allowed(js_devdata, katom->kctx); /* * Remove all atoms on the same context from ringbuffers. This * will not remove atoms that are already on the GPU, as these * are guaranteed not to have fail dependencies on the failed * atom. */ for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) { struct kbase_jd_atom *katom_idx0 = kbase_gpu_inspect(kbdev, i, 0); struct kbase_jd_atom *katom_idx1 = kbase_gpu_inspect(kbdev, i, 1); if (katom_idx0 && katom_idx0->kctx == katom->kctx && HAS_DEP(katom_idx0) && katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { /* Dequeue katom_idx0 from ringbuffer */ kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); if (katom_idx1 && katom_idx1->kctx == katom->kctx && HAS_DEP(katom_idx1) && katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { /* Dequeue katom_idx1 from ringbuffer */ kbase_gpu_dequeue_atom(kbdev, i, end_timestamp); katom_idx1->event_code = BASE_JD_EVENT_STOPPED; kbase_jm_return_atom_to_js(kbdev, katom_idx1); } katom_idx0->event_code = BASE_JD_EVENT_STOPPED; kbase_jm_return_atom_to_js(kbdev, katom_idx0); } else if (katom_idx1 && katom_idx1->kctx == katom->kctx && HAS_DEP(katom_idx1) && katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { /* Can not dequeue this atom yet - will be * dequeued when atom at idx0 completes */ katom_idx1->event_code = BASE_JD_EVENT_STOPPED; kbase_gpu_mark_atom_for_return(kbdev, katom_idx1); } } } KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, js, completion_code); if (job_tail != 0 && job_tail != katom->jc) { bool was_updated = (job_tail != katom->jc); /* Some of the job has been executed, so we update the job chain * address to where we should resume from */ katom->jc = job_tail; if (was_updated) KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, katom, job_tail, js); } /* Only update the event code for jobs that weren't cancelled */ if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) katom->event_code = (base_jd_event_code)completion_code; kbase_device_trace_register_access(kctx, REG_WRITE, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 1 << js); /* Complete the job, and start new ones * * Also defer remaining work onto the workqueue: * - Re-queue Soft-stopped jobs * - For any other jobs, queue the job back into the dependency system * - Schedule out the parent context if necessary, and schedule a new * one in. */ #ifdef CONFIG_GPU_TRACEPOINTS { /* The atom in the HEAD */ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, 0); if (next_katom && next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) { char js_string[16]; trace_gpu_sched_switch(kbasep_make_job_slot_string(js, js_string, sizeof(js_string)), ktime_to_ns(*end_timestamp), (u32)next_katom->kctx->id, 0, next_katom->work_id); kbdev->hwaccess.backend.slot_rb[js].last_context = next_katom->kctx; } else { char js_string[16]; trace_gpu_sched_switch(kbasep_make_job_slot_string(js, js_string, sizeof(js_string)), ktime_to_ns(ktime_get()), 0, 0, 0); kbdev->hwaccess.backend.slot_rb[js].last_context = 0; } } #endif if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET) kbase_reset_gpu_silent(kbdev); if (completion_code == BASE_JD_EVENT_STOPPED) katom = kbase_jm_return_atom_to_js(kbdev, katom); else katom = kbase_jm_complete(kbdev, katom, end_timestamp); if (katom) { /* Cross-slot dependency has now become runnable. Try to submit * it. */ /* Check if there are lower priority jobs to soft stop */ kbase_job_slot_ctx_priority_check_locked(kctx, katom); kbase_jm_try_kick(kbdev, 1 << katom->slot_nr); } /* Job completion may have unblocked other atoms. Try to update all job * slots */ kbase_backend_slot_update(kbdev); } void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) { int js; lockdep_assert_held(&kbdev->hwaccess_lock); /* Reset should always take the GPU out of protected mode */ WARN_ON(kbase_gpu_in_protected_mode(kbdev)); for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { int atom_idx = 0; int idx; for (idx = 0; idx < SLOT_RB_SIZE; idx++) { struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, atom_idx); bool keep_in_jm_rb = false; if (!katom) break; if (katom->protected_state.exit == KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT) { KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_END(kbdev); kbase_vinstr_resume(kbdev->vinstr_ctx); /* protected mode sanity checks */ KBASE_DEBUG_ASSERT_MSG( kbase_jd_katom_is_protected(katom) == kbase_gpu_in_protected_mode(kbdev), "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)", kbase_jd_katom_is_protected(katom), kbase_gpu_in_protected_mode(kbdev)); KBASE_DEBUG_ASSERT_MSG( (kbase_jd_katom_is_protected(katom) && js == 0) || !kbase_jd_katom_is_protected(katom), "Protected atom on JS%d not supported", js); } if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED) keep_in_jm_rb = true; kbase_gpu_release_atom(kbdev, katom, NULL); /* * If the atom wasn't on HW when the reset was issued * then leave it in the RB and next time we're kicked * it will be processed again from the starting state. */ if (keep_in_jm_rb) { kbasep_js_job_check_deref_cores(kbdev, katom); katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; katom->affinity = 0; katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; /* As the atom was not removed, increment the * index so that we read the correct atom in the * next iteration. */ atom_idx++; continue; } /* * The atom was on the HW when the reset was issued * all we can do is fail the atom. */ kbase_gpu_dequeue_atom(kbdev, js, NULL); katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; kbase_jm_complete(kbdev, katom, end_timestamp); } } kbdev->protected_mode_transition = false; } static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev, int js, struct kbase_jd_atom *katom, u32 action) { u32 hw_action = action & JS_COMMAND_MASK; kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action, katom->core_req, katom); katom->kctx->blocked_js[js][katom->sched_priority] = true; } static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom, u32 action, bool disjoint) { katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; kbase_gpu_mark_atom_for_return(kbdev, katom); katom->kctx->blocked_js[katom->slot_nr][katom->sched_priority] = true; if (disjoint) kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom); } static int should_stop_x_dep_slot(struct kbase_jd_atom *katom) { if (katom->x_post_dep) { struct kbase_jd_atom *dep_atom = katom->x_post_dep; if (dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB && dep_atom->gpu_rb_state != KBASE_ATOM_GPU_RB_RETURN_TO_JS) return dep_atom->slot_nr; } return -1; } static void kbase_job_evicted(struct kbase_jd_atom *katom) { kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom, katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT); } bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, struct kbase_context *kctx, int js, struct kbase_jd_atom *katom, u32 action) { struct kbase_jd_atom *katom_idx0; struct kbase_jd_atom *katom_idx1; bool katom_idx0_valid, katom_idx1_valid; bool ret = false; int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1; int prio_idx0 = 0, prio_idx1 = 0; lockdep_assert_held(&kbdev->hwaccess_lock); katom_idx0 = kbase_gpu_inspect(kbdev, js, 0); katom_idx1 = kbase_gpu_inspect(kbdev, js, 1); if (katom_idx0) prio_idx0 = katom_idx0->sched_priority; if (katom_idx1) prio_idx1 = katom_idx1->sched_priority; if (katom) { katom_idx0_valid = (katom_idx0 == katom); /* If idx0 is to be removed and idx1 is on the same context, * then idx1 must also be removed otherwise the atoms might be * returned out of order */ if (katom_idx1) katom_idx1_valid = (katom_idx1 == katom) || (katom_idx0_valid && (katom_idx0->kctx == katom_idx1->kctx)); else katom_idx1_valid = false; } else { katom_idx0_valid = (katom_idx0 && (!kctx || katom_idx0->kctx == kctx)); katom_idx1_valid = (katom_idx1 && (!kctx || katom_idx1->kctx == kctx) && prio_idx0 == prio_idx1); } if (katom_idx0_valid) stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0); if (katom_idx1_valid) stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); if (katom_idx0_valid) { if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { /* Simple case - just dequeue and return */ kbase_gpu_dequeue_atom(kbdev, js, NULL); if (katom_idx1_valid) { kbase_gpu_dequeue_atom(kbdev, js, NULL); katom_idx1->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; kbase_jm_return_atom_to_js(kbdev, katom_idx1); katom_idx1->kctx->blocked_js[js][prio_idx1] = true; } katom_idx0->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; kbase_jm_return_atom_to_js(kbdev, katom_idx0); katom_idx0->kctx->blocked_js[js][prio_idx0] = true; } else { /* katom_idx0 is on GPU */ if (katom_idx1 && katom_idx1->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) { /* katom_idx0 and katom_idx1 are on GPU */ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), NULL) == 0) { /* idx0 has already completed - stop * idx1 if needed*/ if (katom_idx1_valid) { kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); ret = true; } } else { /* idx1 is in NEXT registers - attempt * to remove */ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), JS_COMMAND_NOP, NULL); if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) != 0 || kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) != 0) { /* idx1 removed successfully, * will be handled in IRQ */ kbase_job_evicted(katom_idx1); kbase_gpu_remove_atom(kbdev, katom_idx1, action, true); stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1); /* stop idx0 if still on GPU */ kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); ret = true; } else if (katom_idx1_valid) { /* idx0 has already completed, * stop idx1 if needed */ kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); ret = true; } } } else if (katom_idx1_valid) { /* idx1 not on GPU but must be dequeued*/ /* idx1 will be handled in IRQ */ kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); /* stop idx0 */ /* This will be repeated for anything removed * from the next registers, since their normal * flow was also interrupted, and this function * might not enter disjoint state e.g. if we * don't actually do a hard stop on the head * atom */ kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); ret = true; } else { /* no atom in idx1 */ /* just stop idx0 */ kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); ret = true; } } } else if (katom_idx1_valid) { if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { /* Mark for return */ /* idx1 will be returned once idx0 completes */ kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); } else { /* idx1 is on GPU */ if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), NULL) == 0) { /* idx0 has already completed - stop idx1 */ kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); ret = true; } else { /* idx1 is in NEXT registers - attempt to * remove */ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), JS_COMMAND_NOP, NULL); if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL) != 0 || kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL) != 0) { /* idx1 removed successfully, will be * handled in IRQ once idx0 completes */ kbase_job_evicted(katom_idx1); kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); } else { /* idx0 has already completed - stop * idx1 */ kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); ret = true; } } } } if (stop_x_dep_idx0 != -1) kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0, NULL, action); if (stop_x_dep_idx1 != -1) kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1, NULL, action); return ret; } void kbase_gpu_cacheclean(struct kbase_device *kbdev) { /* Limit the number of loops to avoid a hang if the interrupt is missed */ u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS; mutex_lock(&kbdev->cacheclean_lock); /* use GPU_COMMAND completion solution */ /* clean & invalidate the caches */ KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0); kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CLEAN_INV_CACHES, NULL); /* wait for cache flush to complete before continuing */ while (--max_loops && (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & CLEAN_CACHES_COMPLETED) == 0) ; /* clear the CLEAN_CACHES_COMPLETED irq */ KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, CLEAN_CACHES_COMPLETED); kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), CLEAN_CACHES_COMPLETED, NULL); KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_CLEANING, "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang."); mutex_unlock(&kbdev->cacheclean_lock); } void kbase_backend_cacheclean(struct kbase_device *kbdev, struct kbase_jd_atom *katom) { if (katom->need_cache_flush_cores_retained) { unsigned long flags; kbase_gpu_cacheclean(kbdev); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_pm_unrequest_cores(kbdev, false, katom->need_cache_flush_cores_retained); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); katom->need_cache_flush_cores_retained = 0; } } void kbase_backend_complete_wq(struct kbase_device *kbdev, struct kbase_jd_atom *katom) { /* * If cache flush required due to HW workaround then perform the flush * now */ kbase_backend_cacheclean(kbdev, katom); if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) && (katom->core_req & BASE_JD_REQ_FS) && katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) && !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) { dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n"); if (kbasep_10969_workaround_clamp_coordinates(katom)) { /* The job had a TILE_RANGE_FAULT after was soft-stopped * Due to an HW issue we try to execute the job again. */ dev_dbg(kbdev->dev, "Clamping has been executed, try to rerun the job\n" ); katom->event_code = BASE_JD_EVENT_STOPPED; katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN; } } /* Clear the coreref_state now - while check_deref_cores() may not have * been called yet, the caller will have taken a copy of this field. If * this is not done, then if the atom is re-scheduled (following a soft * stop) then the core reference would not be retaken. */ katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; katom->affinity = 0; } void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, base_jd_core_req core_req, u64 affinity, enum kbase_atom_coreref_state coreref_state) { unsigned long flags; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity, coreref_state); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); if (!kbdev->pm.active_count) { mutex_lock(&kbdev->js_data.runpool_mutex); mutex_lock(&kbdev->pm.lock); kbase_pm_update_active(kbdev); mutex_unlock(&kbdev->pm.lock); mutex_unlock(&kbdev->js_data.runpool_mutex); } } void kbase_gpu_dump_slots(struct kbase_device *kbdev) { struct kbasep_js_device_data *js_devdata; unsigned long flags; int js; js_devdata = &kbdev->js_data; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n"); for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { int idx; for (idx = 0; idx < SLOT_RB_SIZE; idx++) { struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, idx); if (katom) dev_info(kbdev->dev, " js%d idx%d : katom=%p gpu_rb_state=%d\n", js, idx, katom, katom->gpu_rb_state); else dev_info(kbdev->dev, " js%d idx%d : empty\n", js, idx); } } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); }