// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
|
/*
|
*
|
* (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved.
|
*
|
* This program is free software and is provided to you under the terms of the
|
* GNU General Public License version 2 as published by the Free Software
|
* Foundation, and any use by you of this program is subject to the terms
|
* of such GNU license.
|
*
|
* This program is distributed in the hope that it will be useful,
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
* GNU General Public License for more details.
|
*
|
* You should have received a copy of the GNU General Public License
|
* along with this program; if not, you can access it online at
|
* http://www.gnu.org/licenses/gpl-2.0.html.
|
*
|
*/
|
|
#include "mali_kbase_timeline_priv.h"
|
#include "mali_kbase_tlstream.h"
|
#include "mali_kbase_tracepoints.h"
|
#include "mali_kbase_timeline.h"
|
|
#include <device/mali_kbase_device.h>
|
|
#include <linux/poll.h>
|
#include <linux/version_compat_defs.h>
|
#include <linux/anon_inodes.h>
|
|
/* Explicitly include epoll header for old kernels. Not required from 4.16. */
|
#if KERNEL_VERSION(4, 16, 0) > LINUX_VERSION_CODE
|
#include <uapi/linux/eventpoll.h>
|
#endif
|
|
static int kbase_unprivileged_global_profiling;
|
|
/**
|
* kbase_unprivileged_global_profiling_set - set permissions for unprivileged processes
|
*
|
* @val: String containing value to set. Only strings representing positive
|
* integers are accepted as valid; any non-positive integer (including 0)
|
* is rejected.
|
* @kp: Module parameter associated with this method.
|
*
|
* This method can only be used to enable permissions for unprivileged processes,
|
* if they are disabled: for this reason, the only values which are accepted are
|
* strings representing positive integers. Since it's impossible to disable
|
* permissions once they're set, any integer which is non-positive is rejected,
|
* including 0.
|
*
|
* Return: 0 if success, otherwise error code.
|
*/
|
static int kbase_unprivileged_global_profiling_set(const char *val, const struct kernel_param *kp)
|
{
|
int new_val;
|
int ret = kstrtoint(val, 0, &new_val);
|
|
if (ret == 0) {
|
if (new_val < 1)
|
return -EINVAL;
|
|
kbase_unprivileged_global_profiling = 1;
|
}
|
|
return ret;
|
}
|
|
static const struct kernel_param_ops kbase_global_unprivileged_profiling_ops = {
|
.get = param_get_int,
|
.set = kbase_unprivileged_global_profiling_set,
|
};
|
|
module_param_cb(kbase_unprivileged_global_profiling, &kbase_global_unprivileged_profiling_ops,
|
&kbase_unprivileged_global_profiling, 0600);
|
|
/* The timeline stream file operations functions. */
|
static ssize_t kbasep_timeline_io_read(struct file *filp, char __user *buffer,
|
size_t size, loff_t *f_pos);
|
static __poll_t kbasep_timeline_io_poll(struct file *filp, poll_table *wait);
|
static int kbasep_timeline_io_release(struct inode *inode, struct file *filp);
|
static int kbasep_timeline_io_fsync(struct file *filp, loff_t start, loff_t end,
|
int datasync);
|
|
static bool timeline_is_permitted(void)
|
{
|
#if KERNEL_VERSION(5, 8, 0) <= LINUX_VERSION_CODE
|
return kbase_unprivileged_global_profiling || perfmon_capable();
|
#else
|
return kbase_unprivileged_global_profiling || capable(CAP_SYS_ADMIN);
|
#endif
|
}
|
|
/**
|
* kbasep_timeline_io_packet_pending - check timeline streams for pending
|
* packets
|
*
|
* @timeline: Timeline instance
|
* @ready_stream: Pointer to variable where stream will be placed
|
* @rb_idx_raw: Pointer to variable where read buffer index will be placed
|
*
|
* Function checks all streams for pending packets. It will stop as soon as
|
* packet ready to be submitted to user space is detected. Variables under
|
* pointers, passed as the parameters to this function will be updated with
|
* values pointing to right stream and buffer.
|
*
|
* Return: non-zero if any of timeline streams has at last one packet ready
|
*/
|
static int
|
kbasep_timeline_io_packet_pending(struct kbase_timeline *timeline,
|
struct kbase_tlstream **ready_stream,
|
unsigned int *rb_idx_raw)
|
{
|
enum tl_stream_type i;
|
|
KBASE_DEBUG_ASSERT(ready_stream);
|
KBASE_DEBUG_ASSERT(rb_idx_raw);
|
|
for (i = (enum tl_stream_type)0; i < TL_STREAM_TYPE_COUNT; ++i) {
|
struct kbase_tlstream *stream = &timeline->streams[i];
|
*rb_idx_raw = atomic_read(&stream->rbi);
|
/* Read buffer index may be updated by writer in case of
|
* overflow. Read and write buffer indexes must be
|
* loaded in correct order.
|
*/
|
smp_rmb();
|
if (atomic_read(&stream->wbi) != *rb_idx_raw) {
|
*ready_stream = stream;
|
return 1;
|
}
|
}
|
|
return 0;
|
}
|
|
/**
|
* kbasep_timeline_has_header_data() - check timeline headers for pending
|
* packets
|
*
|
* @timeline: Timeline instance
|
*
|
* Return: non-zero if any of timeline headers has at last one packet ready.
|
*/
|
static int kbasep_timeline_has_header_data(struct kbase_timeline *timeline)
|
{
|
return timeline->obj_header_btc || timeline->aux_header_btc
|
#if MALI_USE_CSF
|
|| timeline->csf_tl_reader.tl_header.btc
|
#endif
|
;
|
}
|
|
/**
|
* copy_stream_header() - copy timeline stream header.
|
*
|
* @buffer: Pointer to the buffer provided by user.
|
* @size: Maximum amount of data that can be stored in the buffer.
|
* @copy_len: Pointer to amount of bytes that has been copied already
|
* within the read system call.
|
* @hdr: Pointer to the stream header.
|
* @hdr_size: Header size.
|
* @hdr_btc: Pointer to the remaining number of bytes to copy.
|
*
|
* Return: 0 if success, -1 otherwise.
|
*/
|
static inline int copy_stream_header(char __user *buffer, size_t size,
|
ssize_t *copy_len, const char *hdr,
|
size_t hdr_size, size_t *hdr_btc)
|
{
|
const size_t offset = hdr_size - *hdr_btc;
|
const size_t copy_size = MIN(size - *copy_len, *hdr_btc);
|
|
if (!*hdr_btc)
|
return 0;
|
|
if (WARN_ON(*hdr_btc > hdr_size))
|
return -1;
|
|
if (copy_to_user(&buffer[*copy_len], &hdr[offset], copy_size))
|
return -1;
|
|
*hdr_btc -= copy_size;
|
*copy_len += copy_size;
|
|
return 0;
|
}
|
|
/**
|
* kbasep_timeline_copy_headers - copy timeline headers to the user
|
*
|
* @timeline: Timeline instance
|
* @buffer: Pointer to the buffer provided by user
|
* @size: Maximum amount of data that can be stored in the buffer
|
* @copy_len: Pointer to amount of bytes that has been copied already
|
* within the read system call.
|
*
|
* This helper function checks if timeline headers have not been sent
|
* to the user, and if so, sends them. copy_len is respectively
|
* updated.
|
*
|
* Return: 0 if success, -1 if copy_to_user has failed.
|
*/
|
static inline int kbasep_timeline_copy_headers(struct kbase_timeline *timeline,
|
char __user *buffer, size_t size,
|
ssize_t *copy_len)
|
{
|
if (copy_stream_header(buffer, size, copy_len, obj_desc_header,
|
obj_desc_header_size, &timeline->obj_header_btc))
|
return -1;
|
|
if (copy_stream_header(buffer, size, copy_len, aux_desc_header,
|
aux_desc_header_size, &timeline->aux_header_btc))
|
return -1;
|
#if MALI_USE_CSF
|
if (copy_stream_header(buffer, size, copy_len,
|
timeline->csf_tl_reader.tl_header.data,
|
timeline->csf_tl_reader.tl_header.size,
|
&timeline->csf_tl_reader.tl_header.btc))
|
return -1;
|
#endif
|
return 0;
|
}
|
|
/**
|
* kbasep_timeline_io_read - copy data from streams to buffer provided by user
|
*
|
* @filp: Pointer to file structure
|
* @buffer: Pointer to the buffer provided by user
|
* @size: Maximum amount of data that can be stored in the buffer
|
* @f_pos: Pointer to file offset (unused)
|
*
|
* Return: number of bytes stored in the buffer
|
*/
|
static ssize_t kbasep_timeline_io_read(struct file *filp, char __user *buffer,
|
size_t size, loff_t *f_pos)
|
{
|
ssize_t copy_len = 0;
|
struct kbase_timeline *timeline;
|
|
KBASE_DEBUG_ASSERT(filp);
|
KBASE_DEBUG_ASSERT(f_pos);
|
|
if (WARN_ON(!filp->private_data))
|
return -EFAULT;
|
|
timeline = (struct kbase_timeline *)filp->private_data;
|
|
if (!buffer)
|
return -EINVAL;
|
|
if (*f_pos < 0)
|
return -EINVAL;
|
|
mutex_lock(&timeline->reader_lock);
|
|
while (copy_len < size) {
|
struct kbase_tlstream *stream = NULL;
|
unsigned int rb_idx_raw = 0;
|
unsigned int wb_idx_raw;
|
unsigned int rb_idx;
|
size_t rb_size;
|
|
if (kbasep_timeline_copy_headers(timeline, buffer, size,
|
©_len)) {
|
copy_len = -EFAULT;
|
break;
|
}
|
|
/* If we already read some packets and there is no
|
* packet pending then return back to user.
|
* If we don't have any data yet, wait for packet to be
|
* submitted.
|
*/
|
if (copy_len > 0) {
|
if (!kbasep_timeline_io_packet_pending(
|
timeline, &stream, &rb_idx_raw))
|
break;
|
} else {
|
if (wait_event_interruptible(
|
timeline->event_queue,
|
kbasep_timeline_io_packet_pending(
|
timeline, &stream, &rb_idx_raw))) {
|
copy_len = -ERESTARTSYS;
|
break;
|
}
|
}
|
|
if (WARN_ON(!stream)) {
|
copy_len = -EFAULT;
|
break;
|
}
|
|
/* Check if this packet fits into the user buffer.
|
* If so copy its content.
|
*/
|
rb_idx = rb_idx_raw % PACKET_COUNT;
|
rb_size = atomic_read(&stream->buffer[rb_idx].size);
|
if (rb_size > size - copy_len)
|
break;
|
if (copy_to_user(&buffer[copy_len], stream->buffer[rb_idx].data,
|
rb_size)) {
|
copy_len = -EFAULT;
|
break;
|
}
|
|
/* If the distance between read buffer index and write
|
* buffer index became more than PACKET_COUNT, then overflow
|
* happened and we need to ignore the last portion of bytes
|
* that we have just sent to user.
|
*/
|
smp_rmb();
|
wb_idx_raw = atomic_read(&stream->wbi);
|
|
if (wb_idx_raw - rb_idx_raw < PACKET_COUNT) {
|
copy_len += rb_size;
|
atomic_inc(&stream->rbi);
|
#if MALI_UNIT_TEST
|
atomic_add(rb_size, &timeline->bytes_collected);
|
#endif /* MALI_UNIT_TEST */
|
|
} else {
|
const unsigned int new_rb_idx_raw =
|
wb_idx_raw - PACKET_COUNT + 1;
|
/* Adjust read buffer index to the next valid buffer */
|
atomic_set(&stream->rbi, new_rb_idx_raw);
|
}
|
}
|
|
mutex_unlock(&timeline->reader_lock);
|
|
return copy_len;
|
}
|
|
/**
|
* kbasep_timeline_io_poll - poll timeline stream for packets
|
* @filp: Pointer to file structure
|
* @wait: Pointer to poll table
|
*
|
* Return: EPOLLIN | EPOLLRDNORM if data can be read without blocking,
|
* otherwise zero, or EPOLLHUP | EPOLLERR on error.
|
*/
|
static __poll_t kbasep_timeline_io_poll(struct file *filp, poll_table *wait)
|
{
|
struct kbase_tlstream *stream;
|
unsigned int rb_idx;
|
struct kbase_timeline *timeline;
|
|
KBASE_DEBUG_ASSERT(filp);
|
KBASE_DEBUG_ASSERT(wait);
|
|
if (WARN_ON(!filp->private_data))
|
return EPOLLHUP | EPOLLERR;
|
|
timeline = (struct kbase_timeline *)filp->private_data;
|
|
/* If there are header bytes to copy, read will not block */
|
if (kbasep_timeline_has_header_data(timeline))
|
return EPOLLIN | EPOLLRDNORM;
|
|
poll_wait(filp, &timeline->event_queue, wait);
|
if (kbasep_timeline_io_packet_pending(timeline, &stream, &rb_idx))
|
return EPOLLIN | EPOLLRDNORM;
|
|
return (__poll_t)0;
|
}
|
|
int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags)
|
{
|
/* The timeline stream file operations structure. */
|
static const struct file_operations kbasep_tlstream_fops = {
|
.owner = THIS_MODULE,
|
.release = kbasep_timeline_io_release,
|
.read = kbasep_timeline_io_read,
|
.poll = kbasep_timeline_io_poll,
|
.fsync = kbasep_timeline_io_fsync,
|
};
|
int err;
|
|
if (!timeline_is_permitted())
|
return -EPERM;
|
|
if (WARN_ON(!kbdev) || (flags & ~BASE_TLSTREAM_FLAGS_MASK))
|
return -EINVAL;
|
|
err = kbase_timeline_acquire(kbdev, flags);
|
if (err)
|
return err;
|
|
err = anon_inode_getfd("[mali_tlstream]", &kbasep_tlstream_fops, kbdev->timeline,
|
O_RDONLY | O_CLOEXEC);
|
if (err < 0)
|
kbase_timeline_release(kbdev->timeline);
|
|
return err;
|
}
|
|
#if IS_ENABLED(CONFIG_DEBUG_FS)
|
static int kbasep_timeline_io_open(struct inode *in, struct file *file)
|
{
|
struct kbase_device *const kbdev = in->i_private;
|
|
if (WARN_ON(!kbdev))
|
return -EFAULT;
|
|
file->private_data = kbdev->timeline;
|
return kbase_timeline_acquire(kbdev, BASE_TLSTREAM_FLAGS_MASK &
|
~BASE_TLSTREAM_JOB_DUMPING_ENABLED);
|
}
|
|
void kbase_timeline_io_debugfs_init(struct kbase_device *const kbdev)
|
{
|
static const struct file_operations kbasep_tlstream_debugfs_fops = {
|
.owner = THIS_MODULE,
|
.open = kbasep_timeline_io_open,
|
.release = kbasep_timeline_io_release,
|
.read = kbasep_timeline_io_read,
|
.poll = kbasep_timeline_io_poll,
|
.fsync = kbasep_timeline_io_fsync,
|
};
|
struct dentry *file;
|
|
if (WARN_ON(!kbdev) || WARN_ON(IS_ERR_OR_NULL(kbdev->mali_debugfs_directory)))
|
return;
|
|
file = debugfs_create_file("tlstream", 0400, kbdev->mali_debugfs_directory, kbdev,
|
&kbasep_tlstream_debugfs_fops);
|
|
if (IS_ERR_OR_NULL(file))
|
dev_warn(kbdev->dev, "Unable to create timeline debugfs entry");
|
}
|
#else
|
/*
|
* Stub function for when debugfs is disabled
|
*/
|
void kbase_timeline_io_debugfs_init(struct kbase_device *const kbdev)
|
{
|
}
|
#endif
|
|
/**
|
* kbasep_timeline_io_release - release timeline stream descriptor
|
* @inode: Pointer to inode structure
|
* @filp: Pointer to file structure
|
*
|
* Return: always return zero
|
*/
|
static int kbasep_timeline_io_release(struct inode *inode, struct file *filp)
|
{
|
CSTD_UNUSED(inode);
|
|
kbase_timeline_release(filp->private_data);
|
return 0;
|
}
|
|
static int kbasep_timeline_io_fsync(struct file *filp, loff_t start, loff_t end,
|
int datasync)
|
{
|
CSTD_UNUSED(start);
|
CSTD_UNUSED(end);
|
CSTD_UNUSED(datasync);
|
|
return kbase_timeline_streams_flush(filp->private_data);
|
}
|
