// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
|
/*
|
*
|
* (C) COPYRIGHT 2021-2022 ARM Limited. All rights reserved.
|
*
|
* This program is free software and is provided to you under the terms of the
|
* GNU General Public License version 2 as published by the Free Software
|
* Foundation, and any use by you of this program is subject to the terms
|
* of such GNU license.
|
*
|
* This program is distributed in the hope that it will be useful,
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
* GNU General Public License for more details.
|
*
|
* You should have received a copy of the GNU General Public License
|
* along with this program; if not, you can access it online at
|
* http://www.gnu.org/licenses/gpl-2.0.html.
|
*
|
*/
|
|
#include "hwcnt/mali_kbase_hwcnt_gpu.h"
|
#include "hwcnt/mali_kbase_hwcnt_gpu_narrow.h"
|
|
#include <linux/bug.h>
|
#include <linux/err.h>
|
#include <linux/slab.h>
|
|
int kbase_hwcnt_gpu_metadata_narrow_create(const struct kbase_hwcnt_metadata_narrow **dst_md_narrow,
|
const struct kbase_hwcnt_metadata *src_md)
|
{
|
struct kbase_hwcnt_description desc;
|
struct kbase_hwcnt_group_description group;
|
struct kbase_hwcnt_block_description blks[KBASE_HWCNT_V5_BLOCK_TYPE_COUNT];
|
size_t prfcnt_values_per_block;
|
size_t blk;
|
int err;
|
struct kbase_hwcnt_metadata_narrow *metadata_narrow;
|
|
if (!dst_md_narrow || !src_md || !src_md->grp_metadata ||
|
!src_md->grp_metadata[0].blk_metadata)
|
return -EINVAL;
|
|
/* Only support 1 group count and KBASE_HWCNT_V5_BLOCK_TYPE_COUNT block
|
* count in the metadata.
|
*/
|
if ((kbase_hwcnt_metadata_group_count(src_md) != 1) ||
|
(kbase_hwcnt_metadata_block_count(src_md, 0) != KBASE_HWCNT_V5_BLOCK_TYPE_COUNT))
|
return -EINVAL;
|
|
/* Get the values count in the first block. */
|
prfcnt_values_per_block = kbase_hwcnt_metadata_block_values_count(src_md, 0, 0);
|
|
/* check all blocks should have same values count. */
|
for (blk = 1; blk < KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; blk++) {
|
size_t val_cnt = kbase_hwcnt_metadata_block_values_count(src_md, 0, blk);
|
if (val_cnt != prfcnt_values_per_block)
|
return -EINVAL;
|
}
|
|
/* Only support 64 and 128 entries per block. */
|
if ((prfcnt_values_per_block != 64) && (prfcnt_values_per_block != 128))
|
return -EINVAL;
|
|
metadata_narrow = kmalloc(sizeof(*metadata_narrow), GFP_KERNEL);
|
if (!metadata_narrow)
|
return -ENOMEM;
|
|
/* Narrow to 64 entries per block to keep API backward compatibility. */
|
prfcnt_values_per_block = 64;
|
|
for (blk = 0; blk < KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; blk++) {
|
size_t blk_hdr_cnt = kbase_hwcnt_metadata_block_headers_count(src_md, 0, blk);
|
blks[blk] = (struct kbase_hwcnt_block_description){
|
.type = kbase_hwcnt_metadata_block_type(src_md, 0, blk),
|
.inst_cnt = kbase_hwcnt_metadata_block_instance_count(src_md, 0, blk),
|
.hdr_cnt = blk_hdr_cnt,
|
.ctr_cnt = prfcnt_values_per_block - blk_hdr_cnt,
|
};
|
}
|
|
group = (struct kbase_hwcnt_group_description){
|
.type = kbase_hwcnt_metadata_group_type(src_md, 0),
|
.blk_cnt = KBASE_HWCNT_V5_BLOCK_TYPE_COUNT,
|
.blks = blks,
|
};
|
|
desc = (struct kbase_hwcnt_description){
|
.grp_cnt = kbase_hwcnt_metadata_group_count(src_md),
|
.avail_mask = src_md->avail_mask,
|
.clk_cnt = src_md->clk_cnt,
|
.grps = &group,
|
};
|
|
err = kbase_hwcnt_metadata_create(&desc, &metadata_narrow->metadata);
|
if (!err) {
|
/* Narrow down the buffer size to half as the narrowed metadata
|
* only supports 32-bit but the created metadata uses 64-bit for
|
* block entry.
|
*/
|
metadata_narrow->dump_buf_bytes = metadata_narrow->metadata->dump_buf_bytes >> 1;
|
*dst_md_narrow = metadata_narrow;
|
} else {
|
kfree(metadata_narrow);
|
}
|
|
return err;
|
}
|
|
void kbase_hwcnt_gpu_metadata_narrow_destroy(const struct kbase_hwcnt_metadata_narrow *md_narrow)
|
{
|
if (!md_narrow)
|
return;
|
|
kbase_hwcnt_metadata_destroy(md_narrow->metadata);
|
kfree(md_narrow);
|
}
|
|
int kbase_hwcnt_dump_buffer_narrow_alloc(const struct kbase_hwcnt_metadata_narrow *md_narrow,
|
struct kbase_hwcnt_dump_buffer_narrow *dump_buf)
|
{
|
size_t dump_buf_bytes;
|
size_t clk_cnt_buf_bytes;
|
u8 *buf;
|
|
if (!md_narrow || !dump_buf)
|
return -EINVAL;
|
|
dump_buf_bytes = md_narrow->dump_buf_bytes;
|
clk_cnt_buf_bytes = sizeof(*dump_buf->clk_cnt_buf) * md_narrow->metadata->clk_cnt;
|
|
/* Make a single allocation for both dump_buf and clk_cnt_buf. */
|
buf = kmalloc(dump_buf_bytes + clk_cnt_buf_bytes, GFP_KERNEL);
|
if (!buf)
|
return -ENOMEM;
|
|
*dump_buf = (struct kbase_hwcnt_dump_buffer_narrow){
|
.md_narrow = md_narrow,
|
.dump_buf = (u32 *)buf,
|
.clk_cnt_buf = (u64 *)(buf + dump_buf_bytes),
|
};
|
|
return 0;
|
}
|
|
void kbase_hwcnt_dump_buffer_narrow_free(struct kbase_hwcnt_dump_buffer_narrow *dump_buf_narrow)
|
{
|
if (!dump_buf_narrow)
|
return;
|
|
kfree(dump_buf_narrow->dump_buf);
|
*dump_buf_narrow = (struct kbase_hwcnt_dump_buffer_narrow){ .md_narrow = NULL,
|
.dump_buf = NULL,
|
.clk_cnt_buf = NULL };
|
}
|
|
int kbase_hwcnt_dump_buffer_narrow_array_alloc(
|
const struct kbase_hwcnt_metadata_narrow *md_narrow, size_t n,
|
struct kbase_hwcnt_dump_buffer_narrow_array *dump_bufs)
|
{
|
struct kbase_hwcnt_dump_buffer_narrow *buffers;
|
size_t buf_idx;
|
unsigned int order;
|
unsigned long addr;
|
size_t dump_buf_bytes;
|
size_t clk_cnt_buf_bytes;
|
size_t total_dump_buf_size;
|
|
if (!md_narrow || !dump_bufs)
|
return -EINVAL;
|
|
dump_buf_bytes = md_narrow->dump_buf_bytes;
|
clk_cnt_buf_bytes = sizeof(*dump_bufs->bufs->clk_cnt_buf) * md_narrow->metadata->clk_cnt;
|
|
/* Allocate memory for the dump buffer struct array */
|
buffers = kmalloc_array(n, sizeof(*buffers), GFP_KERNEL);
|
if (!buffers)
|
return -ENOMEM;
|
|
/* Allocate pages for the actual dump buffers, as they tend to be fairly
|
* large.
|
*/
|
order = get_order((dump_buf_bytes + clk_cnt_buf_bytes) * n);
|
addr = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
|
|
if (!addr) {
|
kfree(buffers);
|
return -ENOMEM;
|
}
|
|
*dump_bufs = (struct kbase_hwcnt_dump_buffer_narrow_array){
|
.page_addr = addr,
|
.page_order = order,
|
.buf_cnt = n,
|
.bufs = buffers,
|
};
|
|
total_dump_buf_size = dump_buf_bytes * n;
|
/* Set the buffer of each dump buf */
|
for (buf_idx = 0; buf_idx < n; buf_idx++) {
|
const size_t dump_buf_offset = dump_buf_bytes * buf_idx;
|
const size_t clk_cnt_buf_offset =
|
total_dump_buf_size + (clk_cnt_buf_bytes * buf_idx);
|
|
buffers[buf_idx] = (struct kbase_hwcnt_dump_buffer_narrow){
|
.md_narrow = md_narrow,
|
.dump_buf = (u32 *)(addr + dump_buf_offset),
|
.clk_cnt_buf = (u64 *)(addr + clk_cnt_buf_offset),
|
};
|
}
|
|
return 0;
|
}
|
|
void kbase_hwcnt_dump_buffer_narrow_array_free(
|
struct kbase_hwcnt_dump_buffer_narrow_array *dump_bufs)
|
{
|
if (!dump_bufs)
|
return;
|
|
kfree(dump_bufs->bufs);
|
free_pages(dump_bufs->page_addr, dump_bufs->page_order);
|
memset(dump_bufs, 0, sizeof(*dump_bufs));
|
}
|
|
void kbase_hwcnt_dump_buffer_block_copy_strict_narrow(u32 *dst_blk, const u64 *src_blk,
|
const u64 *blk_em, size_t val_cnt)
|
{
|
size_t val;
|
|
for (val = 0; val < val_cnt; val++) {
|
bool val_enabled = kbase_hwcnt_enable_map_block_value_enabled(blk_em, val);
|
u32 src_val = (src_blk[val] > U32_MAX) ? U32_MAX : (u32)src_blk[val];
|
|
dst_blk[val] = val_enabled ? src_val : 0;
|
}
|
}
|
|
void kbase_hwcnt_dump_buffer_copy_strict_narrow(struct kbase_hwcnt_dump_buffer_narrow *dst_narrow,
|
const struct kbase_hwcnt_dump_buffer *src,
|
const struct kbase_hwcnt_enable_map *dst_enable_map)
|
{
|
const struct kbase_hwcnt_metadata_narrow *metadata_narrow;
|
size_t grp;
|
size_t clk;
|
|
if (WARN_ON(!dst_narrow) || WARN_ON(!src) || WARN_ON(!dst_enable_map) ||
|
WARN_ON(dst_narrow->md_narrow->metadata == src->metadata) ||
|
WARN_ON(dst_narrow->md_narrow->metadata->grp_cnt != src->metadata->grp_cnt) ||
|
WARN_ON(src->metadata->grp_cnt != 1) ||
|
WARN_ON(dst_narrow->md_narrow->metadata->grp_metadata[0].blk_cnt !=
|
src->metadata->grp_metadata[0].blk_cnt) ||
|
WARN_ON(dst_narrow->md_narrow->metadata->grp_metadata[0].blk_cnt !=
|
KBASE_HWCNT_V5_BLOCK_TYPE_COUNT) ||
|
WARN_ON(dst_narrow->md_narrow->metadata->grp_metadata[0].blk_metadata[0].ctr_cnt >
|
src->metadata->grp_metadata[0].blk_metadata[0].ctr_cnt))
|
return;
|
|
/* Don't use src metadata since src buffer is bigger than dst buffer. */
|
metadata_narrow = dst_narrow->md_narrow;
|
|
for (grp = 0; grp < kbase_hwcnt_metadata_narrow_group_count(metadata_narrow); grp++) {
|
size_t blk;
|
size_t blk_cnt = kbase_hwcnt_metadata_narrow_block_count(metadata_narrow, grp);
|
|
for (blk = 0; blk < blk_cnt; blk++) {
|
size_t blk_inst;
|
size_t blk_inst_cnt = kbase_hwcnt_metadata_narrow_block_instance_count(
|
metadata_narrow, grp, blk);
|
|
for (blk_inst = 0; blk_inst < blk_inst_cnt; blk_inst++) {
|
/* The narrowed down buffer is only 32-bit. */
|
u32 *dst_blk = kbase_hwcnt_dump_buffer_narrow_block_instance(
|
dst_narrow, grp, blk, blk_inst);
|
const u64 *src_blk = kbase_hwcnt_dump_buffer_block_instance(
|
src, grp, blk, blk_inst);
|
const u64 *blk_em = kbase_hwcnt_enable_map_block_instance(
|
dst_enable_map, grp, blk, blk_inst);
|
size_t val_cnt = kbase_hwcnt_metadata_narrow_block_values_count(
|
metadata_narrow, grp, blk);
|
/* Align upwards to include padding bytes */
|
val_cnt = KBASE_HWCNT_ALIGN_UPWARDS(
|
val_cnt, (KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT /
|
KBASE_HWCNT_VALUE_BYTES));
|
|
kbase_hwcnt_dump_buffer_block_copy_strict_narrow(dst_blk, src_blk,
|
blk_em, val_cnt);
|
}
|
}
|
}
|
|
for (clk = 0; clk < metadata_narrow->metadata->clk_cnt; clk++) {
|
bool clk_enabled =
|
kbase_hwcnt_clk_enable_map_enabled(dst_enable_map->clk_enable_map, clk);
|
|
dst_narrow->clk_cnt_buf[clk] = clk_enabled ? src->clk_cnt_buf[clk] : 0;
|
}
|
}
|
