From 102a0743326a03cd1a1202ceda21e175b7d3575c Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Tue, 20 Feb 2024 01:20:52 +0000
Subject: [PATCH] add new system file
---
kernel/drivers/pci/msi.c | 238 +++++++++++++++++++++++++++++++++++------------------------
1 files changed, 142 insertions(+), 96 deletions(-)
diff --git a/kernel/drivers/pci/msi.c b/kernel/drivers/pci/msi.c
index 8a74095..a102848 100644
--- a/kernel/drivers/pci/msi.c
+++ b/kernel/drivers/pci/msi.c
@@ -58,8 +58,8 @@
#define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs
#endif
+#ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS
/* Arch hooks */
-
int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
{
struct msi_controller *chip = dev->bus->msi;
@@ -132,6 +132,7 @@
{
return default_teardown_msi_irqs(dev);
}
+#endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */
static void default_restore_msi_irq(struct pci_dev *dev, int irq)
{
@@ -192,6 +193,9 @@
static void __iomem *pci_msix_desc_addr(struct msi_desc *desc)
{
+ if (desc->msi_attrib.is_virtual)
+ return NULL;
+
return desc->mask_base +
desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
}
@@ -206,14 +210,20 @@
u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag)
{
u32 mask_bits = desc->masked;
+ void __iomem *desc_addr;
if (pci_msi_ignore_mask)
+ return 0;
+
+ desc_addr = pci_msix_desc_addr(desc);
+ if (!desc_addr)
return 0;
mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
if (flag & PCI_MSIX_ENTRY_CTRL_MASKBIT)
mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
- writel(mask_bits, pci_msix_desc_addr(desc) + PCI_MSIX_ENTRY_VECTOR_CTRL);
+
+ writel(mask_bits, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
return mask_bits;
}
@@ -237,7 +247,7 @@
}
/**
- * pci_msi_mask_irq - Generic irq chip callback to mask PCI/MSI interrupts
+ * pci_msi_mask_irq - Generic IRQ chip callback to mask PCI/MSI interrupts
* @data: pointer to irqdata associated to that interrupt
*/
void pci_msi_mask_irq(struct irq_data *data)
@@ -247,7 +257,7 @@
EXPORT_SYMBOL_GPL(pci_msi_mask_irq);
/**
- * pci_msi_unmask_irq - Generic irq chip callback to unmask PCI/MSI interrupts
+ * pci_msi_unmask_irq - Generic IRQ chip callback to unmask PCI/MSI interrupts
* @data: pointer to irqdata associated to that interrupt
*/
void pci_msi_unmask_irq(struct irq_data *data)
@@ -272,6 +282,11 @@
if (entry->msi_attrib.is_msix) {
void __iomem *base = pci_msix_desc_addr(entry);
+
+ if (!base) {
+ WARN_ON(1);
+ return;
+ }
msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR);
msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR);
@@ -303,6 +318,9 @@
} else if (entry->msi_attrib.is_msix) {
void __iomem *base = pci_msix_desc_addr(entry);
bool unmasked = !(entry->masked & PCI_MSIX_ENTRY_CTRL_MASKBIT);
+
+ if (!base)
+ goto skip;
/*
* The specification mandates that the entry is masked
@@ -347,7 +365,13 @@
/* Ensure that the writes are visible in the device */
pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
}
+
+skip:
entry->msg = *msg;
+
+ if (entry->write_msi_msg)
+ entry->write_msi_msg(entry, entry->write_msi_msg_data);
+
}
void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg)
@@ -552,15 +576,14 @@
}
static struct msi_desc *
-msi_setup_entry(struct pci_dev *dev, int nvec, const struct irq_affinity *affd)
+msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
{
- struct cpumask *masks = NULL;
+ struct irq_affinity_desc *masks = NULL;
struct msi_desc *entry;
u16 control;
if (affd)
masks = irq_create_affinity_masks(nvec, affd);
-
/* MSI Entry Initialization */
entry = alloc_msi_entry(&dev->dev, nvec, masks);
@@ -574,6 +597,7 @@
entry->msi_attrib.is_msix = 0;
entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT);
+ entry->msi_attrib.is_virtual = 0;
entry->msi_attrib.entry_nr = 0;
entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT);
entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */
@@ -612,16 +636,16 @@
* msi_capability_init - configure device's MSI capability structure
* @dev: pointer to the pci_dev data structure of MSI device function
* @nvec: number of interrupts to allocate
- * @affd: description of automatic irq affinity assignments (may be %NULL)
+ * @affd: description of automatic IRQ affinity assignments (may be %NULL)
*
* Setup the MSI capability structure of the device with the requested
* number of interrupts. A return value of zero indicates the successful
- * setup of an entry with the new MSI irq. A negative return value indicates
+ * setup of an entry with the new MSI IRQ. A negative return value indicates
* an error, and a positive return value indicates the number of interrupts
* which could have been allocated.
*/
static int msi_capability_init(struct pci_dev *dev, int nvec,
- const struct irq_affinity *affd)
+ struct irq_affinity *affd)
{
struct msi_desc *entry;
int ret;
@@ -633,7 +657,7 @@
if (!entry)
return -ENOMEM;
- /* All MSIs are unmasked by default, Mask them all */
+ /* All MSIs are unmasked by default; mask them all */
mask = msi_mask(entry->msi_attrib.multi_cap);
msi_mask_irq(entry, mask, mask);
@@ -661,7 +685,7 @@
return ret;
}
- /* Set MSI enabled bits */
+ /* Set MSI enabled bits */
pci_intx_for_msi(dev, 0);
pci_msi_set_enable(dev, 1);
dev->msi_enabled = 1;
@@ -688,17 +712,18 @@
table_offset &= PCI_MSIX_TABLE_OFFSET;
phys_addr = pci_resource_start(dev, bir) + table_offset;
- return ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
+ return ioremap(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
}
static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
struct msix_entry *entries, int nvec,
- const struct irq_affinity *affd)
+ struct irq_affinity *affd)
{
- struct cpumask *curmsk, *masks = NULL;
+ struct irq_affinity_desc *curmsk, *masks = NULL;
struct msi_desc *entry;
void __iomem *addr;
int ret, i;
+ int vec_count = pci_msix_vec_count(dev);
if (affd)
masks = irq_create_affinity_masks(nvec, affd);
@@ -722,6 +747,10 @@
entry->msi_attrib.entry_nr = entries[i].entry;
else
entry->msi_attrib.entry_nr = i;
+
+ entry->msi_attrib.is_virtual =
+ entry->msi_attrib.entry_nr >= vec_count;
+
entry->msi_attrib.default_irq = dev->irq;
entry->mask_base = base;
@@ -768,14 +797,14 @@
* @dev: pointer to the pci_dev data structure of MSI-X device function
* @entries: pointer to an array of struct msix_entry entries
* @nvec: number of @entries
- * @affd: Optional pointer to enable automatic affinity assignement
+ * @affd: Optional pointer to enable automatic affinity assignment
*
* Setup the MSI-X capability structure of device function with a
- * single MSI-X irq. A return of zero indicates the successful setup of
- * requested MSI-X entries with allocated irqs or non-zero for otherwise.
+ * single MSI-X IRQ. A return of zero indicates the successful setup of
+ * requested MSI-X entries with allocated IRQs or non-zero for otherwise.
**/
static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
- int nvec, const struct irq_affinity *affd)
+ int nvec, struct irq_affinity *affd)
{
void __iomem *base;
int ret, tsize;
@@ -838,7 +867,7 @@
out_avail:
if (ret < 0) {
/*
- * If we had some success, report the number of irqs
+ * If we had some success, report the number of IRQs
* we succeeded in setting up.
*/
struct msi_desc *entry;
@@ -864,7 +893,7 @@
/**
* pci_msi_supported - check whether MSI may be enabled on a device
* @dev: pointer to the pci_dev data structure of MSI device function
- * @nvec: how many MSIs have been requested ?
+ * @nvec: how many MSIs have been requested?
*
* Look at global flags, the device itself, and its parent buses
* to determine if MSI/-X are supported for the device. If MSI/-X is
@@ -878,7 +907,7 @@
if (!pci_msi_enable)
return 0;
- if (!dev || dev->no_msi || dev->current_state != PCI_D0)
+ if (!dev || dev->no_msi)
return 0;
/*
@@ -947,7 +976,7 @@
mask = msi_mask(desc->msi_attrib.multi_cap);
msi_mask_irq(desc, mask, 0);
- /* Restore dev->irq to its default pin-assertion irq */
+ /* Restore dev->irq to its default pin-assertion IRQ */
dev->irq = desc->msi_attrib.default_irq;
pcibios_alloc_irq(dev);
}
@@ -983,18 +1012,18 @@
EXPORT_SYMBOL(pci_msix_vec_count);
static int __pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries,
- int nvec, const struct irq_affinity *affd)
+ int nvec, struct irq_affinity *affd, int flags)
{
int nr_entries;
int i, j;
- if (!pci_msi_supported(dev, nvec))
+ if (!pci_msi_supported(dev, nvec) || dev->current_state != PCI_D0)
return -EINVAL;
nr_entries = pci_msix_vec_count(dev);
if (nr_entries < 0)
return nr_entries;
- if (nvec > nr_entries)
+ if (nvec > nr_entries && !(flags & PCI_IRQ_VIRTUAL))
return nr_entries;
if (entries) {
@@ -1009,7 +1038,7 @@
}
}
- /* Check whether driver already requested for MSI irq */
+ /* Check whether driver already requested for MSI IRQ */
if (dev->msi_enabled) {
pci_info(dev, "can't enable MSI-X (MSI IRQ already assigned)\n");
return -EINVAL;
@@ -1067,15 +1096,15 @@
EXPORT_SYMBOL(pci_msi_enabled);
static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
- const struct irq_affinity *affd)
+ struct irq_affinity *affd)
{
int nvec;
int rc;
- if (!pci_msi_supported(dev, minvec))
+ if (!pci_msi_supported(dev, minvec) || dev->current_state != PCI_D0)
return -EINVAL;
- /* Check whether driver already requested MSI-X irqs */
+ /* Check whether driver already requested MSI-X IRQs */
if (dev->msix_enabled) {
pci_info(dev, "can't enable MSI (MSI-X already enabled)\n");
return -EINVAL;
@@ -1128,7 +1157,8 @@
static int __pci_enable_msix_range(struct pci_dev *dev,
struct msix_entry *entries, int minvec,
- int maxvec, const struct irq_affinity *affd)
+ int maxvec, struct irq_affinity *affd,
+ int flags)
{
int rc, nvec = maxvec;
@@ -1145,7 +1175,7 @@
return -ENOSPC;
}
- rc = __pci_enable_msix(dev, entries, nvec, affd);
+ rc = __pci_enable_msix(dev, entries, nvec, affd, flags);
if (rc == 0)
return nvec;
@@ -1162,8 +1192,8 @@
* pci_enable_msix_range - configure device's MSI-X capability structure
* @dev: pointer to the pci_dev data structure of MSI-X device function
* @entries: pointer to an array of MSI-X entries
- * @minvec: minimum number of MSI-X irqs requested
- * @maxvec: maximum number of MSI-X irqs requested
+ * @minvec: minimum number of MSI-X IRQs requested
+ * @maxvec: maximum number of MSI-X IRQs requested
*
* Setup the MSI-X capability structure of device function with a maximum
* possible number of interrupts in the range between @minvec and @maxvec
@@ -1176,7 +1206,7 @@
int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
int minvec, int maxvec)
{
- return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL);
+ return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0);
}
EXPORT_SYMBOL(pci_enable_msix_range);
@@ -1200,11 +1230,10 @@
*/
int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
unsigned int max_vecs, unsigned int flags,
- const struct irq_affinity *affd)
+ struct irq_affinity *affd)
{
- static const struct irq_affinity msi_default_affd;
- int msix_vecs = -ENOSPC;
- int msi_vecs = -ENOSPC;
+ struct irq_affinity msi_default_affd = {0};
+ int nvecs = -ENOSPC;
if (flags & PCI_IRQ_AFFINITY) {
if (!affd)
@@ -1215,30 +1244,34 @@
}
if (flags & PCI_IRQ_MSIX) {
- msix_vecs = __pci_enable_msix_range(dev, NULL, min_vecs,
- max_vecs, affd);
- if (msix_vecs > 0)
- return msix_vecs;
+ nvecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs,
+ affd, flags);
+ if (nvecs > 0)
+ return nvecs;
}
if (flags & PCI_IRQ_MSI) {
- msi_vecs = __pci_enable_msi_range(dev, min_vecs, max_vecs,
- affd);
- if (msi_vecs > 0)
- return msi_vecs;
+ nvecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd);
+ if (nvecs > 0)
+ return nvecs;
}
- /* use legacy irq if allowed */
+ /* use legacy IRQ if allowed */
if (flags & PCI_IRQ_LEGACY) {
if (min_vecs == 1 && dev->irq) {
+ /*
+ * Invoke the affinity spreading logic to ensure that
+ * the device driver can adjust queue configuration
+ * for the single interrupt case.
+ */
+ if (affd)
+ irq_create_affinity_masks(1, affd);
pci_intx(dev, 1);
return 1;
}
}
- if (msix_vecs == -ENOSPC)
- return -ENOSPC;
- return msi_vecs;
+ return nvecs;
}
EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity);
@@ -1257,19 +1290,24 @@
/**
* pci_irq_vector - return Linux IRQ number of a device vector
- * @dev: PCI device to operate on
- * @nr: device-relative interrupt vector index (0-based).
+ * @dev: PCI device to operate on
+ * @nr: Interrupt vector index (0-based)
+ *
+ * @nr has the following meanings depending on the interrupt mode:
+ * MSI-X: The index in the MSI-X vector table
+ * MSI: The index of the enabled MSI vectors
+ * INTx: Must be 0
+ *
+ * Return: The Linux interrupt number or -EINVAl if @nr is out of range.
*/
int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
{
if (dev->msix_enabled) {
struct msi_desc *entry;
- int i = 0;
for_each_pci_msi_entry(entry, dev) {
- if (i == nr)
+ if (entry->msi_attrib.entry_nr == nr)
return entry->irq;
- i++;
}
WARN_ON_ONCE(1);
return -EINVAL;
@@ -1290,20 +1328,25 @@
EXPORT_SYMBOL(pci_irq_vector);
/**
- * pci_irq_get_affinity - return the affinity of a particular msi vector
+ * pci_irq_get_affinity - return the affinity of a particular MSI vector
* @dev: PCI device to operate on
* @nr: device-relative interrupt vector index (0-based).
+ *
+ * @nr has the following meanings depending on the interrupt mode:
+ * MSI-X: The index in the MSI-X vector table
+ * MSI: The index of the enabled MSI vectors
+ * INTx: Must be 0
+ *
+ * Return: A cpumask pointer or NULL if @nr is out of range
*/
const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
{
if (dev->msix_enabled) {
struct msi_desc *entry;
- int i = 0;
for_each_pci_msi_entry(entry, dev) {
- if (i == nr)
- return entry->affinity;
- i++;
+ if (entry->msi_attrib.entry_nr == nr)
+ return &entry->affinity->mask;
}
WARN_ON_ONCE(1);
return NULL;
@@ -1314,28 +1357,12 @@
nr >= entry->nvec_used))
return NULL;
- return &entry->affinity[nr];
+ return &entry->affinity[nr].mask;
} else {
return cpu_possible_mask;
}
}
EXPORT_SYMBOL(pci_irq_get_affinity);
-
-/**
- * pci_irq_get_node - return the numa node of a particular msi vector
- * @pdev: PCI device to operate on
- * @vec: device-relative interrupt vector index (0-based).
- */
-int pci_irq_get_node(struct pci_dev *pdev, int vec)
-{
- const struct cpumask *mask;
-
- mask = pci_irq_get_affinity(pdev, vec);
- if (mask)
- return local_memory_node(cpu_to_node(cpumask_first(mask)));
- return dev_to_node(&pdev->dev);
-}
-EXPORT_SYMBOL(pci_irq_get_node);
struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc)
{
@@ -1371,16 +1398,16 @@
/**
* pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source
- * @dev: Pointer to the PCI device
- * @desc: Pointer to the msi descriptor
+ * @desc: Pointer to the MSI descriptor
*
* The ID number is only used within the irqdomain.
*/
-irq_hw_number_t pci_msi_domain_calc_hwirq(struct pci_dev *dev,
- struct msi_desc *desc)
+static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc)
{
+ struct pci_dev *dev = msi_desc_to_pci_dev(desc);
+
return (irq_hw_number_t)desc->msi_attrib.entry_nr |
- PCI_DEVID(dev->bus->number, dev->devfn) << 11 |
+ pci_dev_id(dev) << 11 |
(pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27;
}
@@ -1390,7 +1417,8 @@
}
/**
- * pci_msi_domain_check_cap - Verify that @domain supports the capabilities for @dev
+ * pci_msi_domain_check_cap - Verify that @domain supports the capabilities
+ * for @dev
* @domain: The interrupt domain to check
* @info: The domain info for verification
* @dev: The device to check
@@ -1425,17 +1453,12 @@
return error;
}
-#ifdef GENERIC_MSI_DOMAIN_OPS
static void pci_msi_domain_set_desc(msi_alloc_info_t *arg,
struct msi_desc *desc)
{
arg->desc = desc;
- arg->hwirq = pci_msi_domain_calc_hwirq(msi_desc_to_pci_dev(desc),
- desc);
+ arg->hwirq = pci_msi_domain_calc_hwirq(desc);
}
-#else
-#define pci_msi_domain_set_desc NULL
-#endif
static struct msi_domain_ops pci_msi_domain_ops_default = {
.set_desc = pci_msi_domain_set_desc,
@@ -1550,13 +1573,13 @@
u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
{
struct device_node *of_node;
- u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
+ u32 rid = pci_dev_id(pdev);
pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
of_node = irq_domain_get_of_node(domain);
- rid = of_node ? of_msi_map_rid(&pdev->dev, of_node, rid) :
- iort_msi_map_rid(&pdev->dev, rid);
+ rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) :
+ iort_msi_map_id(&pdev->dev, rid);
return rid;
}
@@ -1573,12 +1596,35 @@
struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
{
struct irq_domain *dom;
- u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
+ u32 rid = pci_dev_id(pdev);
pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
- dom = of_msi_map_get_device_domain(&pdev->dev, rid);
+ dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI);
if (!dom)
- dom = iort_get_device_domain(&pdev->dev, rid);
+ dom = iort_get_device_domain(&pdev->dev, rid,
+ DOMAIN_BUS_PCI_MSI);
return dom;
}
+
+/**
+ * pci_dev_has_special_msi_domain - Check whether the device is handled by
+ * a non-standard PCI-MSI domain
+ * @pdev: The PCI device to check.
+ *
+ * Returns: True if the device irqdomain or the bus irqdomain is
+ * non-standard PCI/MSI.
+ */
+bool pci_dev_has_special_msi_domain(struct pci_dev *pdev)
+{
+ struct irq_domain *dom = dev_get_msi_domain(&pdev->dev);
+
+ if (!dom)
+ dom = dev_get_msi_domain(&pdev->bus->dev);
+
+ if (!dom)
+ return true;
+
+ return dom->bus_token != DOMAIN_BUS_PCI_MSI;
+}
+
#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */
--
Gitblit v1.6.2