// SPDX-License-Identifier: GPL-2.0
|
/* Copyright (C) 2018 Google, Inc. */
|
|
#include "gasket_interrupt.h"
|
|
#include "gasket_constants.h"
|
#include "gasket_core.h"
|
#include "gasket_sysfs.h"
|
#include <linux/device.h>
|
#include <linux/interrupt.h>
|
#include <linux/printk.h>
|
#ifdef GASKET_KERNEL_TRACE_SUPPORT
|
#define CREATE_TRACE_POINTS
|
#include <trace/events/gasket_interrupt.h>
|
#else
|
#define trace_gasket_interrupt_event(x, ...)
|
#endif
|
/* Retry attempts if the requested number of interrupts aren't available. */
|
#define MSIX_RETRY_COUNT 3
|
|
/* Instance interrupt management data. */
|
struct gasket_interrupt_data {
|
/* The name associated with this interrupt data. */
|
const char *name;
|
|
/* Interrupt type. See gasket_interrupt_type in gasket_core.h */
|
int type;
|
|
/* The PCI device [if any] associated with the owning device. */
|
struct pci_dev *pci_dev;
|
|
/* Set to 1 if MSI-X has successfully been configred, 0 otherwise. */
|
int msix_configured;
|
|
/* The number of interrupts requested by the owning device. */
|
int num_interrupts;
|
|
/* A pointer to the interrupt descriptor struct for this device. */
|
const struct gasket_interrupt_desc *interrupts;
|
|
/* The index of the bar into which interrupts should be mapped. */
|
int interrupt_bar_index;
|
|
/* The width of a single interrupt in a packed interrupt register. */
|
int pack_width;
|
|
/*
|
* Design-wise, these elements should be bundled together, but
|
* pci_enable_msix's interface requires that they be managed
|
* individually (requires array of struct msix_entry).
|
*/
|
|
/* The number of successfully configured interrupts. */
|
int num_configured;
|
|
/* The MSI-X data for each requested/configured interrupt. */
|
struct msix_entry *msix_entries;
|
|
/* The eventfd "callback" data for each interrupt. */
|
struct eventfd_ctx **eventfd_ctxs;
|
|
/* The number of times each interrupt has been called. */
|
ulong *interrupt_counts;
|
|
/* Linux IRQ number. */
|
int irq;
|
};
|
|
/* Structures to display interrupt counts in sysfs. */
|
enum interrupt_sysfs_attribute_type {
|
ATTR_INTERRUPT_COUNTS,
|
};
|
|
/* Set up device registers for interrupt handling. */
|
static void gasket_interrupt_setup(struct gasket_dev *gasket_dev)
|
{
|
int i;
|
int pack_shift;
|
ulong mask;
|
ulong value;
|
struct gasket_interrupt_data *interrupt_data =
|
gasket_dev->interrupt_data;
|
|
if (!interrupt_data) {
|
dev_dbg(gasket_dev->dev, "Interrupt data is not initialized\n");
|
return;
|
}
|
|
dev_dbg(gasket_dev->dev, "Running interrupt setup\n");
|
|
/* Setup the MSIX table. */
|
|
for (i = 0; i < interrupt_data->num_interrupts; i++) {
|
/*
|
* If the interrupt is not packed, we can write the index into
|
* the register directly. If not, we need to deal with a read-
|
* modify-write and shift based on the packing index.
|
*/
|
dev_dbg(gasket_dev->dev,
|
"Setting up interrupt index %d with index 0x%llx and packing %d\n",
|
interrupt_data->interrupts[i].index,
|
interrupt_data->interrupts[i].reg,
|
interrupt_data->interrupts[i].packing);
|
if (interrupt_data->interrupts[i].packing == UNPACKED) {
|
value = interrupt_data->interrupts[i].index;
|
} else {
|
switch (interrupt_data->interrupts[i].packing) {
|
case PACK_0:
|
pack_shift = 0;
|
break;
|
case PACK_1:
|
pack_shift = interrupt_data->pack_width;
|
break;
|
case PACK_2:
|
pack_shift = 2 * interrupt_data->pack_width;
|
break;
|
case PACK_3:
|
pack_shift = 3 * interrupt_data->pack_width;
|
break;
|
default:
|
dev_dbg(gasket_dev->dev,
|
"Found interrupt description with unknown enum %d\n",
|
interrupt_data->interrupts[i].packing);
|
return;
|
}
|
|
mask = ~(0xFFFF << pack_shift);
|
value = gasket_dev_read_64(gasket_dev,
|
interrupt_data->interrupt_bar_index,
|
interrupt_data->interrupts[i].reg);
|
value &= mask;
|
value |= interrupt_data->interrupts[i].index
|
<< pack_shift;
|
}
|
gasket_dev_write_64(gasket_dev, value,
|
interrupt_data->interrupt_bar_index,
|
interrupt_data->interrupts[i].reg);
|
}
|
}
|
|
static void
|
gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data,
|
int interrupt_index)
|
{
|
struct eventfd_ctx *ctx;
|
|
trace_gasket_interrupt_event(interrupt_data->name, interrupt_index);
|
ctx = interrupt_data->eventfd_ctxs[interrupt_index];
|
if (ctx)
|
eventfd_signal(ctx, 1);
|
|
++(interrupt_data->interrupt_counts[interrupt_index]);
|
}
|
|
static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id)
|
{
|
struct gasket_interrupt_data *interrupt_data = dev_id;
|
int interrupt = -1;
|
int i;
|
|
/* If this linear lookup is a problem, we can maintain a map/hash. */
|
for (i = 0; i < interrupt_data->num_interrupts; i++) {
|
if (interrupt_data->msix_entries[i].vector == irq) {
|
interrupt = interrupt_data->msix_entries[i].entry;
|
break;
|
}
|
}
|
if (interrupt == -1) {
|
pr_err("Received unknown irq %d\n", irq);
|
return IRQ_HANDLED;
|
}
|
gasket_handle_interrupt(interrupt_data, interrupt);
|
return IRQ_HANDLED;
|
}
|
|
static int
|
gasket_interrupt_msix_init(struct gasket_interrupt_data *interrupt_data)
|
{
|
int ret = 1;
|
int i;
|
|
interrupt_data->msix_entries =
|
kcalloc(interrupt_data->num_interrupts,
|
sizeof(*interrupt_data->msix_entries), GFP_KERNEL);
|
if (!interrupt_data->msix_entries)
|
return -ENOMEM;
|
|
for (i = 0; i < interrupt_data->num_interrupts; i++) {
|
interrupt_data->msix_entries[i].entry = i;
|
interrupt_data->msix_entries[i].vector = 0;
|
interrupt_data->eventfd_ctxs[i] = NULL;
|
}
|
|
/* Retry MSIX_RETRY_COUNT times if not enough IRQs are available. */
|
for (i = 0; i < MSIX_RETRY_COUNT && ret > 0; i++)
|
ret = pci_enable_msix_exact(interrupt_data->pci_dev,
|
interrupt_data->msix_entries,
|
interrupt_data->num_interrupts);
|
|
if (ret)
|
return ret > 0 ? -EBUSY : ret;
|
interrupt_data->msix_configured = 1;
|
|
for (i = 0; i < interrupt_data->num_interrupts; i++) {
|
ret = request_irq(interrupt_data->msix_entries[i].vector,
|
gasket_msix_interrupt_handler, 0,
|
interrupt_data->name, interrupt_data);
|
|
if (ret) {
|
dev_err(&interrupt_data->pci_dev->dev,
|
"Cannot get IRQ for interrupt %d, vector %d; "
|
"%d\n",
|
i, interrupt_data->msix_entries[i].vector, ret);
|
return ret;
|
}
|
|
interrupt_data->num_configured++;
|
}
|
|
return 0;
|
}
|
|
/*
|
* On QCM DragonBoard, we exit gasket_interrupt_msix_init() and kernel interrupt
|
* setup code with MSIX vectors masked. This is wrong because nothing else in
|
* the driver will normally touch the MSIX vectors.
|
*
|
* As a temporary hack, force unmasking there.
|
*
|
* TODO: Figure out why QCM kernel doesn't unmask the MSIX vectors, after
|
* gasket_interrupt_msix_init(), and remove this code.
|
*/
|
static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev)
|
{
|
int i;
|
#define MSIX_VECTOR_SIZE 16
|
#define MSIX_MASK_BIT_OFFSET 12
|
#define APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE 0x46800
|
for (i = 0; i < gasket_dev->interrupt_data->num_configured; i++) {
|
/* Check if the MSIX vector is unmasked */
|
ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE +
|
MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE;
|
u32 mask =
|
gasket_dev_read_32(gasket_dev,
|
gasket_dev->interrupt_data->interrupt_bar_index,
|
location);
|
if (!(mask & 1))
|
continue;
|
/* Unmask the msix vector (clear 32 bits) */
|
gasket_dev_write_32(gasket_dev, 0,
|
gasket_dev->interrupt_data->interrupt_bar_index,
|
location);
|
}
|
#undef MSIX_VECTOR_SIZE
|
#undef MSIX_MASK_BIT_OFFSET
|
#undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE
|
}
|
|
static ssize_t interrupt_sysfs_show(struct device *device,
|
struct device_attribute *attr, char *buf)
|
{
|
int i, ret;
|
ssize_t written = 0, total_written = 0;
|
struct gasket_interrupt_data *interrupt_data;
|
struct gasket_dev *gasket_dev;
|
struct gasket_sysfs_attribute *gasket_attr;
|
enum interrupt_sysfs_attribute_type sysfs_type;
|
|
gasket_dev = gasket_sysfs_get_device_data(device);
|
if (!gasket_dev) {
|
dev_dbg(device, "No sysfs mapping found for device\n");
|
return 0;
|
}
|
|
gasket_attr = gasket_sysfs_get_attr(device, attr);
|
if (!gasket_attr) {
|
dev_dbg(device, "No sysfs attr data found for device\n");
|
gasket_sysfs_put_device_data(device, gasket_dev);
|
return 0;
|
}
|
|
sysfs_type = (enum interrupt_sysfs_attribute_type)
|
gasket_attr->data.attr_type;
|
interrupt_data = gasket_dev->interrupt_data;
|
switch (sysfs_type) {
|
case ATTR_INTERRUPT_COUNTS:
|
for (i = 0; i < interrupt_data->num_interrupts; ++i) {
|
written =
|
scnprintf(buf, PAGE_SIZE - total_written,
|
"0x%02x: %ld\n", i,
|
interrupt_data->interrupt_counts[i]);
|
total_written += written;
|
buf += written;
|
}
|
ret = total_written;
|
break;
|
default:
|
dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n",
|
attr->attr.name);
|
ret = 0;
|
break;
|
}
|
|
gasket_sysfs_put_attr(device, gasket_attr);
|
gasket_sysfs_put_device_data(device, gasket_dev);
|
return ret;
|
}
|
|
static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = {
|
GASKET_SYSFS_RO(interrupt_counts, interrupt_sysfs_show,
|
ATTR_INTERRUPT_COUNTS),
|
GASKET_END_OF_ATTR_ARRAY,
|
};
|
|
int gasket_interrupt_init(struct gasket_dev *gasket_dev)
|
{
|
int ret;
|
struct gasket_interrupt_data *interrupt_data;
|
const struct gasket_driver_desc *driver_desc =
|
gasket_get_driver_desc(gasket_dev);
|
|
interrupt_data = kzalloc(sizeof(*interrupt_data), GFP_KERNEL);
|
if (!interrupt_data)
|
return -ENOMEM;
|
gasket_dev->interrupt_data = interrupt_data;
|
interrupt_data->name = driver_desc->name;
|
interrupt_data->type = driver_desc->interrupt_type;
|
interrupt_data->pci_dev = gasket_dev->pci_dev;
|
interrupt_data->num_interrupts = driver_desc->num_interrupts;
|
interrupt_data->interrupts = driver_desc->interrupts;
|
interrupt_data->interrupt_bar_index = driver_desc->interrupt_bar_index;
|
interrupt_data->pack_width = driver_desc->interrupt_pack_width;
|
interrupt_data->num_configured = 0;
|
|
interrupt_data->eventfd_ctxs =
|
kcalloc(driver_desc->num_interrupts,
|
sizeof(*interrupt_data->eventfd_ctxs), GFP_KERNEL);
|
if (!interrupt_data->eventfd_ctxs) {
|
kfree(interrupt_data);
|
return -ENOMEM;
|
}
|
|
interrupt_data->interrupt_counts =
|
kcalloc(driver_desc->num_interrupts,
|
sizeof(*interrupt_data->interrupt_counts), GFP_KERNEL);
|
if (!interrupt_data->interrupt_counts) {
|
kfree(interrupt_data->eventfd_ctxs);
|
kfree(interrupt_data);
|
return -ENOMEM;
|
}
|
|
switch (interrupt_data->type) {
|
case PCI_MSIX:
|
ret = gasket_interrupt_msix_init(interrupt_data);
|
if (ret)
|
break;
|
force_msix_interrupt_unmasking(gasket_dev);
|
break;
|
|
default:
|
ret = -EINVAL;
|
}
|
|
if (ret) {
|
/* Failing to setup interrupts will cause the device to report
|
* GASKET_STATUS_LAMED. But it is not fatal.
|
*/
|
dev_warn(gasket_dev->dev,
|
"Couldn't initialize interrupts: %d\n", ret);
|
return 0;
|
}
|
|
gasket_interrupt_setup(gasket_dev);
|
gasket_sysfs_create_entries(gasket_dev->dev_info.device,
|
interrupt_sysfs_attrs);
|
|
return 0;
|
}
|
|
static void
|
gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data)
|
{
|
int i;
|
|
for (i = 0; i < interrupt_data->num_configured; i++)
|
free_irq(interrupt_data->msix_entries[i].vector,
|
interrupt_data);
|
interrupt_data->num_configured = 0;
|
|
if (interrupt_data->msix_configured)
|
pci_disable_msix(interrupt_data->pci_dev);
|
interrupt_data->msix_configured = 0;
|
kfree(interrupt_data->msix_entries);
|
}
|
|
int gasket_interrupt_reinit(struct gasket_dev *gasket_dev)
|
{
|
int ret;
|
|
if (!gasket_dev->interrupt_data) {
|
dev_dbg(gasket_dev->dev,
|
"Attempted to reinit uninitialized interrupt data\n");
|
return -EINVAL;
|
}
|
|
switch (gasket_dev->interrupt_data->type) {
|
case PCI_MSIX:
|
gasket_interrupt_msix_cleanup(gasket_dev->interrupt_data);
|
ret = gasket_interrupt_msix_init(gasket_dev->interrupt_data);
|
if (ret)
|
break;
|
force_msix_interrupt_unmasking(gasket_dev);
|
break;
|
|
default:
|
ret = -EINVAL;
|
}
|
|
if (ret) {
|
/* Failing to setup interrupts will cause the device
|
* to report GASKET_STATUS_LAMED, but is not fatal.
|
*/
|
dev_warn(gasket_dev->dev, "Couldn't reinit interrupts: %d\n",
|
ret);
|
return 0;
|
}
|
|
gasket_interrupt_setup(gasket_dev);
|
|
return 0;
|
}
|
|
/* See gasket_interrupt.h for description. */
|
int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev)
|
{
|
dev_dbg(gasket_dev->dev, "Clearing interrupt counts\n");
|
memset(gasket_dev->interrupt_data->interrupt_counts, 0,
|
gasket_dev->interrupt_data->num_interrupts *
|
sizeof(*gasket_dev->interrupt_data->interrupt_counts));
|
return 0;
|
}
|
|
/* See gasket_interrupt.h for description. */
|
void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev)
|
{
|
struct gasket_interrupt_data *interrupt_data =
|
gasket_dev->interrupt_data;
|
/*
|
* It is possible to get an error code from gasket_interrupt_init
|
* before interrupt_data has been allocated, so check it.
|
*/
|
if (!interrupt_data)
|
return;
|
|
switch (interrupt_data->type) {
|
case PCI_MSIX:
|
gasket_interrupt_msix_cleanup(interrupt_data);
|
break;
|
|
default:
|
break;
|
}
|
|
kfree(interrupt_data->interrupt_counts);
|
kfree(interrupt_data->eventfd_ctxs);
|
kfree(interrupt_data);
|
gasket_dev->interrupt_data = NULL;
|
}
|
|
int gasket_interrupt_system_status(struct gasket_dev *gasket_dev)
|
{
|
if (!gasket_dev->interrupt_data) {
|
dev_dbg(gasket_dev->dev, "Interrupt data is null\n");
|
return GASKET_STATUS_DEAD;
|
}
|
|
if (gasket_dev->interrupt_data->num_configured !=
|
gasket_dev->interrupt_data->num_interrupts) {
|
dev_dbg(gasket_dev->dev,
|
"Not all interrupts were configured\n");
|
return GASKET_STATUS_LAMED;
|
}
|
|
return GASKET_STATUS_ALIVE;
|
}
|
|
int gasket_interrupt_set_eventfd(struct gasket_interrupt_data *interrupt_data,
|
int interrupt, int event_fd)
|
{
|
struct eventfd_ctx *ctx;
|
|
if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
|
return -EINVAL;
|
|
ctx = eventfd_ctx_fdget(event_fd);
|
|
if (IS_ERR(ctx))
|
return PTR_ERR(ctx);
|
|
interrupt_data->eventfd_ctxs[interrupt] = ctx;
|
return 0;
|
}
|
|
int gasket_interrupt_clear_eventfd(struct gasket_interrupt_data *interrupt_data,
|
int interrupt)
|
{
|
if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
|
return -EINVAL;
|
|
if (interrupt_data->eventfd_ctxs[interrupt]) {
|
eventfd_ctx_put(interrupt_data->eventfd_ctxs[interrupt]);
|
interrupt_data->eventfd_ctxs[interrupt] = NULL;
|
}
|
return 0;
|
}
|
