/*
|
*
|
* (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
|
*
|
* This program is free software and is provided to you under the terms of the
|
* GNU General Public License version 2 as published by the Free Software
|
* Foundation, and any use by you of this program is subject to the terms
|
* of such GNU licence.
|
*
|
* A copy of the licence is included with the program, and can also be obtained
|
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
|
* Boston, MA 02110-1301, USA.
|
*
|
*/
|
|
|
|
|
|
/*
|
* Base kernel affinity manager APIs
|
*/
|
|
#include <mali_kbase.h>
|
#include "mali_kbase_js_affinity.h"
|
#include "mali_kbase_hw.h"
|
|
#include <backend/gpu/mali_kbase_pm_internal.h>
|
|
|
bool kbase_js_can_run_job_on_slot_no_lock(struct kbase_device *kbdev,
|
int js)
|
{
|
/*
|
* Here are the reasons for using job slot 2:
|
* - BASE_HW_ISSUE_8987 (which is entirely used for that purpose)
|
* - In absence of the above, then:
|
* - Atoms with BASE_JD_REQ_COHERENT_GROUP
|
* - But, only when there aren't contexts with
|
* KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, because the atoms that run on
|
* all cores on slot 1 could be blocked by those using a coherent group
|
* on slot 2
|
* - And, only when you actually have 2 or more coregroups - if you
|
* only have 1 coregroup, then having jobs for slot 2 implies they'd
|
* also be for slot 1, meaning you'll get interference from them. Jobs
|
* able to run on slot 2 could also block jobs that can only run on
|
* slot 1 (tiler jobs)
|
*/
|
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
|
return true;
|
|
if (js != 2)
|
return true;
|
|
/* Only deal with js==2 now: */
|
if (kbdev->gpu_props.num_core_groups > 1) {
|
/* Only use slot 2 in the 2+ coregroup case */
|
if (kbasep_js_ctx_attr_is_attr_on_runpool(kbdev,
|
KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) ==
|
false) {
|
/* ...But only when we *don't* have atoms that run on
|
* all cores */
|
|
/* No specific check for BASE_JD_REQ_COHERENT_GROUP
|
* atoms - the policy will sort that out */
|
return true;
|
}
|
}
|
|
/* Above checks failed mean we shouldn't use slot 2 */
|
return false;
|
}
|
|
/*
|
* As long as it has been decided to have a deeper modification of
|
* what job scheduler, power manager and affinity manager will
|
* implement, this function is just an intermediate step that
|
* assumes:
|
* - all working cores will be powered on when this is called.
|
* - largest current configuration is 2 core groups.
|
* - It has been decided not to have hardcoded values so the low
|
* and high cores in a core split will be evently distributed.
|
* - Odd combinations of core requirements have been filtered out
|
* and do not get to this function (e.g. CS+T+NSS is not
|
* supported here).
|
* - This function is frequently called and can be optimized,
|
* (see notes in loops), but as the functionallity will likely
|
* be modified, optimization has not been addressed.
|
*/
|
bool kbase_js_choose_affinity(u64 * const affinity,
|
struct kbase_device *kbdev,
|
struct kbase_jd_atom *katom, int js)
|
{
|
base_jd_core_req core_req = katom->core_req;
|
unsigned int num_core_groups = kbdev->gpu_props.num_core_groups;
|
u64 core_availability_mask;
|
|
lockdep_assert_held(&kbdev->hwaccess_lock);
|
|
core_availability_mask = kbase_pm_ca_get_core_mask(kbdev);
|
|
/*
|
* If no cores are currently available (core availability policy is
|
* transitioning) then fail.
|
*/
|
if (0 == core_availability_mask) {
|
*affinity = 0;
|
return false;
|
}
|
|
KBASE_DEBUG_ASSERT(js >= 0);
|
|
if ((core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) ==
|
BASE_JD_REQ_T) {
|
/* If the hardware supports XAFFINITY then we'll only enable
|
* the tiler (which is the default so this is a no-op),
|
* otherwise enable shader core 0. */
|
if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY))
|
*affinity = 1;
|
else
|
*affinity = 0;
|
|
return true;
|
}
|
|
if (1 == kbdev->gpu_props.num_cores) {
|
/* trivial case only one core, nothing to do */
|
*affinity = core_availability_mask &
|
kbdev->pm.debug_core_mask[js];
|
} else {
|
if ((core_req & (BASE_JD_REQ_COHERENT_GROUP |
|
BASE_JD_REQ_SPECIFIC_COHERENT_GROUP))) {
|
if (js == 0 || num_core_groups == 1) {
|
/* js[0] and single-core-group systems just get
|
* the first core group */
|
*affinity =
|
kbdev->gpu_props.props.coherency_info.group[0].core_mask
|
& core_availability_mask &
|
kbdev->pm.debug_core_mask[js];
|
} else {
|
/* js[1], js[2] use core groups 0, 1 for
|
* dual-core-group systems */
|
u32 core_group_idx = ((u32) js) - 1;
|
|
KBASE_DEBUG_ASSERT(core_group_idx <
|
num_core_groups);
|
*affinity =
|
kbdev->gpu_props.props.coherency_info.group[core_group_idx].core_mask
|
& core_availability_mask &
|
kbdev->pm.debug_core_mask[js];
|
|
/* If the job is specifically targeting core
|
* group 1 and the core availability policy is
|
* keeping that core group off, then fail */
|
if (*affinity == 0 && core_group_idx == 1 &&
|
kbdev->pm.backend.cg1_disabled
|
== true)
|
katom->event_code =
|
BASE_JD_EVENT_PM_EVENT;
|
}
|
} else {
|
/* All cores are available when no core split is
|
* required */
|
*affinity = core_availability_mask &
|
kbdev->pm.debug_core_mask[js];
|
}
|
}
|
|
/*
|
* If no cores are currently available in the desired core group(s)
|
* (core availability policy is transitioning) then fail.
|
*/
|
if (*affinity == 0)
|
return false;
|
|
/* Enable core 0 if tiler required for hardware without XAFFINITY
|
* support (notes above) */
|
if (core_req & BASE_JD_REQ_T) {
|
if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY))
|
*affinity = *affinity | 1;
|
}
|
|
return true;
|
}
|
|
static inline bool kbase_js_affinity_is_violating(
|
struct kbase_device *kbdev,
|
u64 *affinities)
|
{
|
/* This implementation checks whether the two slots involved in Generic
|
* thread creation have intersecting affinity. This is due to micro-
|
* architectural issues where a job in slot A targetting cores used by
|
* slot B could prevent the job in slot B from making progress until the
|
* job in slot A has completed.
|
*/
|
u64 affinity_set_left;
|
u64 affinity_set_right;
|
u64 intersection;
|
|
KBASE_DEBUG_ASSERT(affinities != NULL);
|
|
affinity_set_left = affinities[1];
|
|
affinity_set_right = affinities[2];
|
|
/* A violation occurs when any bit in the left_set is also in the
|
* right_set */
|
intersection = affinity_set_left & affinity_set_right;
|
|
return (bool) (intersection != (u64) 0u);
|
}
|
|
bool kbase_js_affinity_would_violate(struct kbase_device *kbdev, int js,
|
u64 affinity)
|
{
|
struct kbasep_js_device_data *js_devdata;
|
u64 new_affinities[BASE_JM_MAX_NR_SLOTS];
|
|
KBASE_DEBUG_ASSERT(kbdev != NULL);
|
KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS);
|
js_devdata = &kbdev->js_data;
|
|
memcpy(new_affinities, js_devdata->runpool_irq.slot_affinities,
|
sizeof(js_devdata->runpool_irq.slot_affinities));
|
|
new_affinities[js] |= affinity;
|
|
return kbase_js_affinity_is_violating(kbdev, new_affinities);
|
}
|
|
void kbase_js_affinity_retain_slot_cores(struct kbase_device *kbdev, int js,
|
u64 affinity)
|
{
|
struct kbasep_js_device_data *js_devdata;
|
u64 cores;
|
|
KBASE_DEBUG_ASSERT(kbdev != NULL);
|
KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS);
|
js_devdata = &kbdev->js_data;
|
|
KBASE_DEBUG_ASSERT(kbase_js_affinity_would_violate(kbdev, js, affinity)
|
== false);
|
|
cores = affinity;
|
while (cores) {
|
int bitnum = fls64(cores) - 1;
|
u64 bit = 1ULL << bitnum;
|
s8 cnt;
|
|
cnt =
|
++(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]);
|
|
if (cnt == 1)
|
js_devdata->runpool_irq.slot_affinities[js] |= bit;
|
|
cores &= ~bit;
|
}
|
}
|
|
void kbase_js_affinity_release_slot_cores(struct kbase_device *kbdev, int js,
|
u64 affinity)
|
{
|
struct kbasep_js_device_data *js_devdata;
|
u64 cores;
|
|
KBASE_DEBUG_ASSERT(kbdev != NULL);
|
KBASE_DEBUG_ASSERT(js < BASE_JM_MAX_NR_SLOTS);
|
js_devdata = &kbdev->js_data;
|
|
cores = affinity;
|
while (cores) {
|
int bitnum = fls64(cores) - 1;
|
u64 bit = 1ULL << bitnum;
|
s8 cnt;
|
|
KBASE_DEBUG_ASSERT(
|
js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum] > 0);
|
|
cnt =
|
--(js_devdata->runpool_irq.slot_affinity_refcount[js][bitnum]);
|
|
if (0 == cnt)
|
js_devdata->runpool_irq.slot_affinities[js] &= ~bit;
|
|
cores &= ~bit;
|
}
|
}
|
|
#if KBASE_TRACE_ENABLE
|
void kbase_js_debug_log_current_affinities(struct kbase_device *kbdev)
|
{
|
struct kbasep_js_device_data *js_devdata;
|
int slot_nr;
|
|
KBASE_DEBUG_ASSERT(kbdev != NULL);
|
js_devdata = &kbdev->js_data;
|
|
for (slot_nr = 0; slot_nr < 3; ++slot_nr)
|
KBASE_TRACE_ADD_SLOT_INFO(kbdev, JS_AFFINITY_CURRENT, NULL,
|
NULL, 0u, slot_nr,
|
(u32) js_devdata->runpool_irq.slot_affinities[slot_nr]);
|
}
|
#endif /* KBASE_TRACE_ENABLE */
|
