// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
|
/*
|
*
|
* (C) COPYRIGHT 2020-2022 ARM Limited. All rights reserved.
|
*
|
* This program is free software and is provided to you under the terms of the
|
* GNU General Public License version 2 as published by the Free Software
|
* Foundation, and any use by you of this program is subject to the terms
|
* of such GNU license.
|
*
|
* This program is distributed in the hope that it will be useful,
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
* GNU General Public License for more details.
|
*
|
* You should have received a copy of the GNU General Public License
|
* along with this program; if not, you can access it online at
|
* http://www.gnu.org/licenses/gpl-2.0.html.
|
*
|
*/
|
|
/*
|
* Implementation of the GPU clock rate trace manager.
|
*/
|
|
#include <mali_kbase.h>
|
#include <mali_kbase_config_defaults.h>
|
#include <linux/clk.h>
|
#include <linux/pm_opp.h>
|
#include <asm/div64.h>
|
#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
|
|
#ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY
|
#include <trace/events/power_gpu_frequency.h>
|
#else
|
#include "mali_power_gpu_frequency_trace.h"
|
#endif
|
|
#ifndef CLK_RATE_TRACE_OPS
|
#define CLK_RATE_TRACE_OPS (NULL)
|
#endif
|
|
/**
|
* get_clk_rate_trace_callbacks() - Returns pointer to clk trace ops.
|
* @kbdev: Pointer to kbase device, used to check if arbitration is enabled
|
* when compiled with arbiter support.
|
* Return: Pointer to clk trace ops if supported or NULL.
|
*/
|
static struct kbase_clk_rate_trace_op_conf *
|
get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev)
|
{
|
/* base case */
|
struct kbase_clk_rate_trace_op_conf *callbacks =
|
(struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS;
|
#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF)
|
const void *arbiter_if_node;
|
|
if (WARN_ON(!kbdev) || WARN_ON(!kbdev->dev))
|
return callbacks;
|
|
arbiter_if_node =
|
of_get_property(kbdev->dev->of_node, "arbiter_if", NULL);
|
/* Arbitration enabled, override the callback pointer.*/
|
if (arbiter_if_node)
|
callbacks = &arb_clk_rate_trace_ops;
|
else
|
dev_dbg(kbdev->dev,
|
"Arbitration supported but disabled by platform. Leaving clk rate callbacks as default.\n");
|
|
#endif
|
|
return callbacks;
|
}
|
|
static int gpu_clk_rate_change_notifier(struct notifier_block *nb,
|
unsigned long event, void *data)
|
{
|
struct kbase_gpu_clk_notifier_data *ndata = data;
|
struct kbase_clk_data *clk_data =
|
container_of(nb, struct kbase_clk_data, clk_rate_change_nb);
|
struct kbase_clk_rate_trace_manager *clk_rtm = clk_data->clk_rtm;
|
unsigned long flags;
|
|
if (WARN_ON_ONCE(clk_data->gpu_clk_handle != ndata->gpu_clk_handle))
|
return NOTIFY_BAD;
|
|
spin_lock_irqsave(&clk_rtm->lock, flags);
|
if (event == POST_RATE_CHANGE) {
|
if (!clk_rtm->gpu_idle &&
|
(clk_data->clock_val != ndata->new_rate)) {
|
kbase_clk_rate_trace_manager_notify_all(
|
clk_rtm, clk_data->index, ndata->new_rate);
|
}
|
|
clk_data->clock_val = ndata->new_rate;
|
}
|
spin_unlock_irqrestore(&clk_rtm->lock, flags);
|
|
return NOTIFY_DONE;
|
}
|
|
static int gpu_clk_data_init(struct kbase_device *kbdev,
|
void *gpu_clk_handle, unsigned int index)
|
{
|
struct kbase_clk_rate_trace_op_conf *callbacks;
|
struct kbase_clk_data *clk_data;
|
struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
|
int ret = 0;
|
|
callbacks = get_clk_rate_trace_callbacks(kbdev);
|
|
if (WARN_ON(!callbacks) ||
|
WARN_ON(!gpu_clk_handle) ||
|
WARN_ON(index >= BASE_MAX_NR_CLOCKS_REGULATORS))
|
return -EINVAL;
|
|
clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
|
if (!clk_data) {
|
dev_err(kbdev->dev, "Failed to allocate data for clock enumerated at index %u", index);
|
return -ENOMEM;
|
}
|
|
clk_data->index = (u8)index;
|
clk_data->gpu_clk_handle = gpu_clk_handle;
|
/* Store the initial value of clock */
|
clk_data->clock_val =
|
callbacks->get_gpu_clk_rate(kbdev, gpu_clk_handle);
|
|
{
|
/* At the initialization time, GPU is powered off. */
|
unsigned long flags;
|
|
spin_lock_irqsave(&clk_rtm->lock, flags);
|
kbase_clk_rate_trace_manager_notify_all(
|
clk_rtm, clk_data->index, 0);
|
spin_unlock_irqrestore(&clk_rtm->lock, flags);
|
}
|
|
clk_data->clk_rtm = clk_rtm;
|
clk_rtm->clks[index] = clk_data;
|
|
clk_data->clk_rate_change_nb.notifier_call =
|
gpu_clk_rate_change_notifier;
|
|
if (callbacks->gpu_clk_notifier_register)
|
ret = callbacks->gpu_clk_notifier_register(kbdev,
|
gpu_clk_handle, &clk_data->clk_rate_change_nb);
|
if (ret) {
|
dev_err(kbdev->dev, "Failed to register notifier for clock enumerated at index %u", index);
|
kfree(clk_data);
|
}
|
|
return ret;
|
}
|
|
int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev)
|
{
|
struct kbase_clk_rate_trace_op_conf *callbacks;
|
struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
|
unsigned int i;
|
int ret = 0;
|
|
callbacks = get_clk_rate_trace_callbacks(kbdev);
|
|
spin_lock_init(&clk_rtm->lock);
|
INIT_LIST_HEAD(&clk_rtm->listeners);
|
|
/* Return early if no callbacks provided for clock rate tracing */
|
if (!callbacks) {
|
WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
|
return 0;
|
}
|
|
clk_rtm->gpu_idle = true;
|
|
for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
|
void *gpu_clk_handle =
|
callbacks->enumerate_gpu_clk(kbdev, i);
|
|
if (!gpu_clk_handle)
|
break;
|
|
ret = gpu_clk_data_init(kbdev, gpu_clk_handle, i);
|
if (ret)
|
goto error;
|
}
|
|
/* Activate clock rate trace manager if at least one GPU clock was
|
* enumerated.
|
*/
|
if (i) {
|
WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks);
|
} else {
|
dev_info(kbdev->dev, "No clock(s) available for rate tracing");
|
WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
|
}
|
|
return 0;
|
|
error:
|
while (i--) {
|
clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister(
|
kbdev, clk_rtm->clks[i]->gpu_clk_handle,
|
&clk_rtm->clks[i]->clk_rate_change_nb);
|
kfree(clk_rtm->clks[i]);
|
}
|
|
return ret;
|
}
|
|
void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev)
|
{
|
struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
|
unsigned int i;
|
|
WARN_ON(!list_empty(&clk_rtm->listeners));
|
|
if (!clk_rtm->clk_rate_trace_ops)
|
return;
|
|
for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
|
if (!clk_rtm->clks[i])
|
break;
|
|
if (clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister)
|
clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister
|
(kbdev, clk_rtm->clks[i]->gpu_clk_handle,
|
&clk_rtm->clks[i]->clk_rate_change_nb);
|
kfree(clk_rtm->clks[i]);
|
}
|
|
WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
|
}
|
|
void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev)
|
{
|
struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
|
unsigned int i;
|
unsigned long flags;
|
|
if (!clk_rtm->clk_rate_trace_ops)
|
return;
|
|
spin_lock_irqsave(&clk_rtm->lock, flags);
|
|
for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
|
struct kbase_clk_data *clk_data = clk_rtm->clks[i];
|
|
if (!clk_data)
|
break;
|
|
if (unlikely(!clk_data->clock_val))
|
continue;
|
|
kbase_clk_rate_trace_manager_notify_all(
|
clk_rtm, clk_data->index, clk_data->clock_val);
|
}
|
|
clk_rtm->gpu_idle = false;
|
spin_unlock_irqrestore(&clk_rtm->lock, flags);
|
}
|
|
void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev)
|
{
|
struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
|
unsigned int i;
|
unsigned long flags;
|
|
if (!clk_rtm->clk_rate_trace_ops)
|
return;
|
|
spin_lock_irqsave(&clk_rtm->lock, flags);
|
|
for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
|
struct kbase_clk_data *clk_data = clk_rtm->clks[i];
|
|
if (!clk_data)
|
break;
|
|
if (unlikely(!clk_data->clock_val))
|
continue;
|
|
kbase_clk_rate_trace_manager_notify_all(
|
clk_rtm, clk_data->index, 0);
|
}
|
|
clk_rtm->gpu_idle = true;
|
spin_unlock_irqrestore(&clk_rtm->lock, flags);
|
}
|
|
void kbase_clk_rate_trace_manager_notify_all(
|
struct kbase_clk_rate_trace_manager *clk_rtm,
|
u32 clk_index,
|
unsigned long new_rate)
|
{
|
struct kbase_clk_rate_listener *pos;
|
struct kbase_device *kbdev;
|
|
lockdep_assert_held(&clk_rtm->lock);
|
|
kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm);
|
|
dev_dbg(kbdev->dev, "%s - GPU clock %u rate changed to %lu, pid: %d",
|
__func__, clk_index, new_rate, current->pid);
|
|
/* Raise standard `power/gpu_frequency` ftrace event */
|
{
|
unsigned long new_rate_khz = new_rate;
|
|
#if BITS_PER_LONG == 64
|
do_div(new_rate_khz, 1000);
|
#elif BITS_PER_LONG == 32
|
new_rate_khz /= 1000;
|
#else
|
#error "unsigned long division is not supported for this architecture"
|
#endif
|
|
trace_gpu_frequency(new_rate_khz, clk_index);
|
}
|
|
/* Notify the listeners. */
|
list_for_each_entry(pos, &clk_rtm->listeners, node) {
|
pos->notify(pos, clk_index, new_rate);
|
}
|
}
|
KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all);
|
