/*
|
*
|
* (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved.
|
*
|
* This program is free software and is provided to you under the terms of the
|
* GNU General Public License version 2 as published by the Free Software
|
* Foundation, and any use by you of this program is subject to the terms
|
* of such GNU licence.
|
*
|
* A copy of the licence is included with the program, and can also be obtained
|
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
|
* Boston, MA 02110-1301, USA.
|
*
|
*/
|
|
|
|
|
#include <mali_kbase.h>
|
#include <mali_kbase_config.h>
|
|
/*
|
* Private functions follow
|
*/
|
|
/**
|
* @brief Check whether a ctx has a certain attribute, and if so, retain that
|
* attribute on the runpool.
|
*
|
* Requires:
|
* - jsctx mutex
|
* - runpool_irq spinlock
|
* - ctx is scheduled on the runpool
|
*
|
* @return true indicates a change in ctx attributes state of the runpool.
|
* In this state, the scheduler might be able to submit more jobs than
|
* previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock()
|
* or similar is called sometime later.
|
* @return false indicates no change in ctx attributes state of the runpool.
|
*/
|
static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
|
{
|
struct kbasep_js_device_data *js_devdata;
|
struct kbasep_js_kctx_info *js_kctx_info;
|
bool runpool_state_changed = false;
|
|
KBASE_DEBUG_ASSERT(kbdev != NULL);
|
KBASE_DEBUG_ASSERT(kctx != NULL);
|
KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
|
js_devdata = &kbdev->js_data;
|
js_kctx_info = &kctx->jctx.sched_info;
|
|
lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
|
lockdep_assert_held(&kbdev->hwaccess_lock);
|
|
KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
|
|
if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) {
|
KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] < S8_MAX);
|
++(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]);
|
|
if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 1) {
|
/* First refcount indicates a state change */
|
runpool_state_changed = true;
|
KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_RUNPOOL, kctx, NULL, 0u, attribute);
|
}
|
}
|
|
return runpool_state_changed;
|
}
|
|
/**
|
* @brief Check whether a ctx has a certain attribute, and if so, release that
|
* attribute on the runpool.
|
*
|
* Requires:
|
* - jsctx mutex
|
* - runpool_irq spinlock
|
* - ctx is scheduled on the runpool
|
*
|
* @return true indicates a change in ctx attributes state of the runpool.
|
* In this state, the scheduler might be able to submit more jobs than
|
* previously, and so the caller should ensure kbasep_js_try_run_next_job_nolock()
|
* or similar is called sometime later.
|
* @return false indicates no change in ctx attributes state of the runpool.
|
*/
|
static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
|
{
|
struct kbasep_js_device_data *js_devdata;
|
struct kbasep_js_kctx_info *js_kctx_info;
|
bool runpool_state_changed = false;
|
|
KBASE_DEBUG_ASSERT(kbdev != NULL);
|
KBASE_DEBUG_ASSERT(kctx != NULL);
|
KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
|
js_devdata = &kbdev->js_data;
|
js_kctx_info = &kctx->jctx.sched_info;
|
|
lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
|
lockdep_assert_held(&kbdev->hwaccess_lock);
|
KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
|
|
if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, attribute) != false) {
|
KBASE_DEBUG_ASSERT(js_devdata->runpool_irq.ctx_attr_ref_count[attribute] > 0);
|
--(js_devdata->runpool_irq.ctx_attr_ref_count[attribute]);
|
|
if (js_devdata->runpool_irq.ctx_attr_ref_count[attribute] == 0) {
|
/* Last de-refcount indicates a state change */
|
runpool_state_changed = true;
|
KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_RUNPOOL, kctx, NULL, 0u, attribute);
|
}
|
}
|
|
return runpool_state_changed;
|
}
|
|
/**
|
* @brief Retain a certain attribute on a ctx, also retaining it on the runpool
|
* if the context is scheduled.
|
*
|
* Requires:
|
* - jsctx mutex
|
* - If the context is scheduled, then runpool_irq spinlock must also be held
|
*
|
* @return true indicates a change in ctx attributes state of the runpool.
|
* This may allow the scheduler to submit more jobs than previously.
|
* @return false indicates no change in ctx attributes state of the runpool.
|
*/
|
static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
|
{
|
struct kbasep_js_kctx_info *js_kctx_info;
|
bool runpool_state_changed = false;
|
|
KBASE_DEBUG_ASSERT(kbdev != NULL);
|
KBASE_DEBUG_ASSERT(kctx != NULL);
|
KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
|
js_kctx_info = &kctx->jctx.sched_info;
|
|
lockdep_assert_held(&kbdev->hwaccess_lock);
|
lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
|
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] < U32_MAX);
|
|
++(js_kctx_info->ctx.ctx_attr_ref_count[attribute]);
|
|
if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
|
/* Only ref-count the attribute on the runpool for the first time this contexts sees this attribute */
|
KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_ON_CTX, kctx, NULL, 0u, attribute);
|
runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, attribute);
|
}
|
|
return runpool_state_changed;
|
}
|
|
/*
|
* @brief Release a certain attribute on a ctx, also releasing it from the runpool
|
* if the context is scheduled.
|
*
|
* Requires:
|
* - jsctx mutex
|
* - If the context is scheduled, then runpool_irq spinlock must also be held
|
*
|
* @return true indicates a change in ctx attributes state of the runpool.
|
* This may allow the scheduler to submit more jobs than previously.
|
* @return false indicates no change in ctx attributes state of the runpool.
|
*/
|
static bool kbasep_js_ctx_attr_ctx_release_attr(struct kbase_device *kbdev, struct kbase_context *kctx, enum kbasep_js_ctx_attr attribute)
|
{
|
struct kbasep_js_kctx_info *js_kctx_info;
|
bool runpool_state_changed = false;
|
|
KBASE_DEBUG_ASSERT(kbdev != NULL);
|
KBASE_DEBUG_ASSERT(kctx != NULL);
|
KBASE_DEBUG_ASSERT(attribute < KBASEP_JS_CTX_ATTR_COUNT);
|
js_kctx_info = &kctx->jctx.sched_info;
|
|
lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
|
KBASE_DEBUG_ASSERT(js_kctx_info->ctx.ctx_attr_ref_count[attribute] > 0);
|
|
if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && js_kctx_info->ctx.ctx_attr_ref_count[attribute] == 1) {
|
lockdep_assert_held(&kbdev->hwaccess_lock);
|
/* Only de-ref-count the attribute on the runpool when this is the last ctx-reference to it */
|
runpool_state_changed = kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, attribute);
|
KBASE_TRACE_ADD(kbdev, JS_CTX_ATTR_NOW_OFF_CTX, kctx, NULL, 0u, attribute);
|
}
|
|
/* De-ref must happen afterwards, because kbasep_js_ctx_attr_runpool_release() needs to check it too */
|
--(js_kctx_info->ctx.ctx_attr_ref_count[attribute]);
|
|
return runpool_state_changed;
|
}
|
|
/*
|
* More commonly used public functions
|
*/
|
|
void kbasep_js_ctx_attr_set_initial_attrs(struct kbase_device *kbdev, struct kbase_context *kctx)
|
{
|
bool runpool_state_changed = false;
|
|
KBASE_DEBUG_ASSERT(kbdev != NULL);
|
KBASE_DEBUG_ASSERT(kctx != NULL);
|
|
if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
|
/* This context never submits, so don't track any scheduling attributes */
|
return;
|
}
|
|
/* Transfer attributes held in the context flags for contexts that have submit enabled */
|
|
/* ... More attributes can be added here ... */
|
|
/* The context should not have been scheduled yet, so ASSERT if this caused
|
* runpool state changes (note that other threads *can't* affect the value
|
* of runpool_state_changed, due to how it's calculated) */
|
KBASE_DEBUG_ASSERT(runpool_state_changed == false);
|
CSTD_UNUSED(runpool_state_changed);
|
}
|
|
void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
|
{
|
bool runpool_state_changed;
|
int i;
|
|
/* Retain any existing attributes */
|
for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) {
|
if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) {
|
/* The context is being scheduled in, so update the runpool with the new attributes */
|
runpool_state_changed = kbasep_js_ctx_attr_runpool_retain_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i);
|
|
/* We don't need to know about state changed, because retaining a
|
* context occurs on scheduling it, and that itself will also try
|
* to run new atoms */
|
CSTD_UNUSED(runpool_state_changed);
|
}
|
}
|
}
|
|
bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
|
{
|
bool runpool_state_changed = false;
|
int i;
|
|
/* Release any existing attributes */
|
for (i = 0; i < KBASEP_JS_CTX_ATTR_COUNT; ++i) {
|
if (kbasep_js_ctx_attr_is_attr_on_ctx(kctx, (enum kbasep_js_ctx_attr) i) != false) {
|
/* The context is being scheduled out, so update the runpool on the removed attributes */
|
runpool_state_changed |= kbasep_js_ctx_attr_runpool_release_attr(kbdev, kctx, (enum kbasep_js_ctx_attr) i);
|
}
|
}
|
|
return runpool_state_changed;
|
}
|
|
void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom)
|
{
|
bool runpool_state_changed = false;
|
base_jd_core_req core_req;
|
|
KBASE_DEBUG_ASSERT(katom);
|
core_req = katom->core_req;
|
|
if (core_req & BASE_JD_REQ_ONLY_COMPUTE)
|
runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE);
|
else
|
runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE);
|
|
if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) {
|
/* Atom that can run on slot1 or slot2, and can use all cores */
|
runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES);
|
}
|
|
/* We don't need to know about state changed, because retaining an
|
* atom occurs on adding it, and that itself will also try to run
|
* new atoms */
|
CSTD_UNUSED(runpool_state_changed);
|
}
|
|
bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state)
|
{
|
bool runpool_state_changed = false;
|
base_jd_core_req core_req;
|
|
KBASE_DEBUG_ASSERT(katom_retained_state);
|
core_req = katom_retained_state->core_req;
|
|
/* No-op for invalid atoms */
|
if (kbasep_js_atom_retained_state_is_valid(katom_retained_state) == false)
|
return false;
|
|
if (core_req & BASE_JD_REQ_ONLY_COMPUTE)
|
runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE);
|
else
|
runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_NON_COMPUTE);
|
|
if ((core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T)) != 0 && (core_req & (BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP)) == 0) {
|
/* Atom that can run on slot1 or slot2, and can use all cores */
|
runpool_state_changed |= kbasep_js_ctx_attr_ctx_release_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES);
|
}
|
|
return runpool_state_changed;
|
}
|
