From 072de836f53be56a70cecf70b43ae43b7ce17376 Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Mon, 11 Dec 2023 10:08:36 +0000
Subject: [PATCH] mk-rootfs.sh
---
kernel/drivers/iommu/mtk_iommu.c | 796 +++++++++++++++++++++++++++++++++++++++-----------------
1 files changed, 555 insertions(+), 241 deletions(-)
diff --git a/kernel/drivers/iommu/mtk_iommu.c b/kernel/drivers/iommu/mtk_iommu.c
index 8e75f34..46f6414 100644
--- a/kernel/drivers/iommu/mtk_iommu.c
+++ b/kernel/drivers/iommu/mtk_iommu.c
@@ -1,21 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2015-2016 MediaTek Inc.
* Author: Yong Wu <yong.wu@mediatek.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
*/
-#include <linux/bootmem.h>
+#include <linux/bitfield.h>
#include <linux/bug.h>
#include <linux/clk.h>
#include <linux/component.h>
#include <linux/device.h>
+#include <linux/dma-direct.h>
#include <linux/dma-iommu.h>
#include <linux/err.h>
#include <linux/interrupt.h>
@@ -23,19 +16,24 @@
#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/list.h>
+#include <linux/mfd/syscon.h>
#include <linux/of_address.h>
#include <linux/of_iommu.h>
#include <linux/of_irq.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
+#include <linux/soc/mediatek/infracfg.h>
#include <asm/barrier.h>
#include <soc/mediatek/smi.h>
#include "mtk_iommu.h"
#define REG_MMU_PT_BASE_ADDR 0x000
+#define MMU_PT_ADDR_MASK GENMASK(31, 7)
#define REG_MMU_INVALIDATE 0x020
#define F_ALL_INVLD 0x2
@@ -44,20 +42,23 @@
#define REG_MMU_INVLD_START_A 0x024
#define REG_MMU_INVLD_END_A 0x028
-#define REG_MMU_INV_SEL 0x038
+#define REG_MMU_INV_SEL_GEN2 0x02c
+#define REG_MMU_INV_SEL_GEN1 0x038
#define F_INVLD_EN0 BIT(0)
#define F_INVLD_EN1 BIT(1)
-#define REG_MMU_STANDARD_AXI_MODE 0x048
+#define REG_MMU_MISC_CTRL 0x048
+#define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17))
+#define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19))
+
#define REG_MMU_DCM_DIS 0x050
+#define REG_MMU_WR_LEN_CTRL 0x054
+#define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21))
#define REG_MMU_CTRL_REG 0x110
+#define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4)
#define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4)
-#define F_MMU_TF_PROTECT_SEL_SHIFT(data) \
- ((data)->m4u_plat == M4U_MT2712 ? 4 : 5)
-/* It's named by F_MMU_TF_PROT_SEL in mt2712. */
-#define F_MMU_TF_PROTECT_SEL(prot, data) \
- (((prot) & 0x3) << F_MMU_TF_PROTECT_SEL_SHIFT(data))
+#define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5)
#define REG_MMU_IVRP_PADDR 0x114
@@ -74,46 +75,70 @@
#define F_INT_CLR_BIT BIT(12)
#define REG_MMU_INT_MAIN_CONTROL 0x124
-#define F_INT_TRANSLATION_FAULT BIT(0)
-#define F_INT_MAIN_MULTI_HIT_FAULT BIT(1)
-#define F_INT_INVALID_PA_FAULT BIT(2)
-#define F_INT_ENTRY_REPLACEMENT_FAULT BIT(3)
-#define F_INT_TLB_MISS_FAULT BIT(4)
-#define F_INT_MISS_TRANSACTION_FIFO_FAULT BIT(5)
-#define F_INT_PRETETCH_TRANSATION_FIFO_FAULT BIT(6)
+ /* mmu0 | mmu1 */
+#define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7))
+#define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8))
+#define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9))
+#define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10))
+#define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11))
+#define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12))
+#define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13))
#define REG_MMU_CPE_DONE 0x12C
#define REG_MMU_FAULT_ST1 0x134
+#define F_REG_MMU0_FAULT_MASK GENMASK(6, 0)
+#define F_REG_MMU1_FAULT_MASK GENMASK(13, 7)
-#define REG_MMU_FAULT_VA 0x13c
+#define REG_MMU0_FAULT_VA 0x13c
+#define F_MMU_INVAL_VA_31_12_MASK GENMASK(31, 12)
+#define F_MMU_INVAL_VA_34_32_MASK GENMASK(11, 9)
+#define F_MMU_INVAL_PA_34_32_MASK GENMASK(8, 6)
#define F_MMU_FAULT_VA_WRITE_BIT BIT(1)
#define F_MMU_FAULT_VA_LAYER_BIT BIT(0)
-#define REG_MMU_INVLD_PA 0x140
-#define REG_MMU_INT_ID 0x150
-#define F_MMU0_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7)
-#define F_MMU0_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f)
+#define REG_MMU0_INVLD_PA 0x140
+#define REG_MMU1_FAULT_VA 0x144
+#define REG_MMU1_INVLD_PA 0x148
+#define REG_MMU0_INT_ID 0x150
+#define REG_MMU1_INT_ID 0x154
+#define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0x7)
+#define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3)
+#define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7)
+#define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f)
-#define MTK_PROTECT_PA_ALIGN 128
+#define MTK_PROTECT_PA_ALIGN 256
-/*
- * Get the local arbiter ID and the portid within the larb arbiter
- * from mtk_m4u_id which is defined by MTK_M4U_ID.
- */
-#define MTK_M4U_TO_LARB(id) (((id) >> 5) & 0xf)
-#define MTK_M4U_TO_PORT(id) ((id) & 0x1f)
+#define HAS_4GB_MODE BIT(0)
+/* HW will use the EMI clock if there isn't the "bclk". */
+#define HAS_BCLK BIT(1)
+#define HAS_VLD_PA_RNG BIT(2)
+#define RESET_AXI BIT(3)
+#define OUT_ORDER_WR_EN BIT(4)
+#define HAS_SUB_COMM BIT(5)
+#define WR_THROT_EN BIT(6)
+#define HAS_LEGACY_IVRP_PADDR BIT(7)
+#define IOVA_34_EN BIT(8)
+
+#define MTK_IOMMU_HAS_FLAG(pdata, _x) \
+ ((((pdata)->flags) & (_x)) == (_x))
struct mtk_iommu_domain {
- spinlock_t pgtlock; /* lock for page table */
-
struct io_pgtable_cfg cfg;
struct io_pgtable_ops *iop;
+ struct mtk_iommu_data *data;
struct iommu_domain domain;
};
-static struct iommu_ops mtk_iommu_ops;
+static const struct iommu_ops mtk_iommu_ops;
+
+static int mtk_iommu_hw_init(const struct mtk_iommu_data *data);
+
+#define MTK_IOMMU_TLB_ADDR(iova) ({ \
+ dma_addr_t _addr = iova; \
+ ((lower_32_bits(_addr) & GENMASK(31, 12)) | upper_32_bits(_addr));\
+})
/*
* In M4U 4GB mode, the physical address is remapped as below:
@@ -137,11 +162,30 @@
* 'E', the CPU physical address keep as is.
* Additionally, The iommu consumers always use the CPU phyiscal address.
*/
-#define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x40000000
+#define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL
static LIST_HEAD(m4ulist); /* List all the M4U HWs */
#define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list)
+
+struct mtk_iommu_iova_region {
+ dma_addr_t iova_base;
+ unsigned long long size;
+};
+
+static const struct mtk_iommu_iova_region single_domain[] = {
+ {.iova_base = 0, .size = SZ_4G},
+};
+
+static const struct mtk_iommu_iova_region mt8192_multi_dom[] = {
+ { .iova_base = 0x0, .size = SZ_4G}, /* disp: 0 ~ 4G */
+ #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)
+ { .iova_base = SZ_4G, .size = SZ_4G}, /* vdec: 4G ~ 8G */
+ { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* CAM/MDP: 8G ~ 12G */
+ { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */
+ { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */
+ #endif
+};
/*
* There may be 1 or 2 M4U HWs, But we always expect they are in the same domain
@@ -165,90 +209,108 @@
return container_of(dom, struct mtk_iommu_domain, domain);
}
-static void mtk_iommu_tlb_flush_all(void *cookie)
+static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data)
{
- struct mtk_iommu_data *data = cookie;
-
for_each_m4u(data) {
+ if (pm_runtime_get_if_in_use(data->dev) <= 0)
+ continue;
+
writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
- data->base + REG_MMU_INV_SEL);
+ data->base + data->plat_data->inv_sel_reg);
writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
wmb(); /* Make sure the tlb flush all done */
+
+ pm_runtime_put(data->dev);
}
}
-static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size,
- size_t granule, bool leaf,
- void *cookie)
+static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size,
+ size_t granule,
+ struct mtk_iommu_data *data)
{
- struct mtk_iommu_data *data = cookie;
-
- for_each_m4u(data) {
- writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
- data->base + REG_MMU_INV_SEL);
-
- writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A);
- writel_relaxed(iova + size - 1,
- data->base + REG_MMU_INVLD_END_A);
- writel_relaxed(F_MMU_INV_RANGE,
- data->base + REG_MMU_INVALIDATE);
- data->tlb_flush_active = true;
- }
-}
-
-static void mtk_iommu_tlb_sync(void *cookie)
-{
- struct mtk_iommu_data *data = cookie;
+ bool has_pm = !!data->dev->pm_domain;
+ unsigned long flags;
int ret;
u32 tmp;
for_each_m4u(data) {
- /* Avoid timing out if there's nothing to wait for */
- if (!data->tlb_flush_active)
- return;
+ if (has_pm) {
+ if (pm_runtime_get_if_in_use(data->dev) <= 0)
+ continue;
+ }
+ spin_lock_irqsave(&data->tlb_lock, flags);
+ writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
+ data->base + data->plat_data->inv_sel_reg);
+
+ writel_relaxed(MTK_IOMMU_TLB_ADDR(iova),
+ data->base + REG_MMU_INVLD_START_A);
+ writel_relaxed(MTK_IOMMU_TLB_ADDR(iova + size - 1),
+ data->base + REG_MMU_INVLD_END_A);
+ writel_relaxed(F_MMU_INV_RANGE,
+ data->base + REG_MMU_INVALIDATE);
+
+ /* tlb sync */
ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,
- tmp, tmp != 0, 10, 100000);
+ tmp, tmp != 0, 10, 1000);
if (ret) {
dev_warn(data->dev,
"Partial TLB flush timed out, falling back to full flush\n");
- mtk_iommu_tlb_flush_all(cookie);
+ mtk_iommu_tlb_flush_all(data);
}
/* Clear the CPE status */
writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
- data->tlb_flush_active = false;
+ spin_unlock_irqrestore(&data->tlb_lock, flags);
+
+ if (has_pm)
+ pm_runtime_put(data->dev);
}
}
-
-static const struct iommu_gather_ops mtk_iommu_gather_ops = {
- .tlb_flush_all = mtk_iommu_tlb_flush_all,
- .tlb_add_flush = mtk_iommu_tlb_add_flush_nosync,
- .tlb_sync = mtk_iommu_tlb_sync,
-};
static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
{
struct mtk_iommu_data *data = dev_id;
struct mtk_iommu_domain *dom = data->m4u_dom;
- u32 int_state, regval, fault_iova, fault_pa;
- unsigned int fault_larb, fault_port;
+ unsigned int fault_larb, fault_port, sub_comm = 0;
+ u32 int_state, regval, va34_32, pa34_32;
+ u64 fault_iova, fault_pa;
bool layer, write;
/* Read error info from registers */
int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1);
- fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA);
+ if (int_state & F_REG_MMU0_FAULT_MASK) {
+ regval = readl_relaxed(data->base + REG_MMU0_INT_ID);
+ fault_iova = readl_relaxed(data->base + REG_MMU0_FAULT_VA);
+ fault_pa = readl_relaxed(data->base + REG_MMU0_INVLD_PA);
+ } else {
+ regval = readl_relaxed(data->base + REG_MMU1_INT_ID);
+ fault_iova = readl_relaxed(data->base + REG_MMU1_FAULT_VA);
+ fault_pa = readl_relaxed(data->base + REG_MMU1_INVLD_PA);
+ }
layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT;
write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT;
- fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA);
- regval = readl_relaxed(data->base + REG_MMU_INT_ID);
- fault_larb = F_MMU0_INT_ID_LARB_ID(regval);
- fault_port = F_MMU0_INT_ID_PORT_ID(regval);
+ if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN)) {
+ va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova);
+ pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova);
+ fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK;
+ fault_iova |= (u64)va34_32 << 32;
+ fault_pa |= (u64)pa34_32 << 32;
+ }
+
+ fault_port = F_MMU_INT_ID_PORT_ID(regval);
+ if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM)) {
+ fault_larb = F_MMU_INT_ID_COMM_ID(regval);
+ sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval);
+ } else {
+ fault_larb = F_MMU_INT_ID_LARB_ID(regval);
+ }
+ fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm];
if (report_iommu_fault(&dom->domain, data->dev, fault_iova,
write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) {
dev_err_ratelimited(
data->dev,
- "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d layer=%d %s\n",
+ "fault type=0x%x iova=0x%llx pa=0x%llx larb=%d port=%d layer=%d %s\n",
int_state, fault_iova, fault_pa, fault_larb, fault_port,
layer, write ? "write" : "read");
}
@@ -263,21 +325,57 @@
return IRQ_HANDLED;
}
-static void mtk_iommu_config(struct mtk_iommu_data *data,
- struct device *dev, bool enable)
+static int mtk_iommu_get_domain_id(struct device *dev,
+ const struct mtk_iommu_plat_data *plat_data)
+{
+ const struct mtk_iommu_iova_region *rgn = plat_data->iova_region;
+ const struct bus_dma_region *dma_rgn = dev->dma_range_map;
+ int i, candidate = -1;
+ dma_addr_t dma_end;
+
+ if (!dma_rgn || plat_data->iova_region_nr == 1)
+ return 0;
+
+ dma_end = dma_rgn->dma_start + dma_rgn->size - 1;
+ for (i = 0; i < plat_data->iova_region_nr; i++, rgn++) {
+ /* Best fit. */
+ if (dma_rgn->dma_start == rgn->iova_base &&
+ dma_end == rgn->iova_base + rgn->size - 1)
+ return i;
+ /* ok if it is inside this region. */
+ if (dma_rgn->dma_start >= rgn->iova_base &&
+ dma_end < rgn->iova_base + rgn->size)
+ candidate = i;
+ }
+
+ if (candidate >= 0)
+ return candidate;
+ dev_err(dev, "Can NOT find the iommu domain id(%pad 0x%llx).\n",
+ &dma_rgn->dma_start, dma_rgn->size);
+ return -EINVAL;
+}
+
+static void mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev,
+ bool enable, unsigned int domid)
{
struct mtk_smi_larb_iommu *larb_mmu;
unsigned int larbid, portid;
- struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+ const struct mtk_iommu_iova_region *region;
int i;
for (i = 0; i < fwspec->num_ids; ++i) {
larbid = MTK_M4U_TO_LARB(fwspec->ids[i]);
portid = MTK_M4U_TO_PORT(fwspec->ids[i]);
- larb_mmu = &data->smi_imu.larb_imu[larbid];
- dev_dbg(dev, "%s iommu port: %d\n",
- enable ? "enable" : "disable", portid);
+ larb_mmu = &data->larb_imu[larbid];
+
+ region = data->plat_data->iova_region + domid;
+ larb_mmu->bank[portid] = upper_32_bits(region->iova_base);
+
+ dev_dbg(dev, "%s iommu for larb(%s) port %d dom %d bank %d.\n",
+ enable ? "enable" : "disable", dev_name(larb_mmu->dev),
+ portid, domid, larb_mmu->bank[portid]);
if (enable)
larb_mmu->mmu |= MTK_SMI_MMU_EN(portid);
@@ -286,25 +384,33 @@
}
}
-static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom)
+static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom,
+ struct mtk_iommu_data *data,
+ unsigned int domid)
{
- struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
+ const struct mtk_iommu_iova_region *region;
- spin_lock_init(&dom->pgtlock);
+ /* Use the exist domain as there is only one pgtable here. */
+ if (data->m4u_dom) {
+ dom->iop = data->m4u_dom->iop;
+ dom->cfg = data->m4u_dom->cfg;
+ dom->domain.pgsize_bitmap = data->m4u_dom->cfg.pgsize_bitmap;
+ goto update_iova_region;
+ }
dom->cfg = (struct io_pgtable_cfg) {
.quirks = IO_PGTABLE_QUIRK_ARM_NS |
IO_PGTABLE_QUIRK_NO_PERMS |
- IO_PGTABLE_QUIRK_TLBI_ON_MAP,
+ IO_PGTABLE_QUIRK_ARM_MTK_EXT,
.pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap,
- .ias = 32,
- .oas = 32,
- .tlb = &mtk_iommu_gather_ops,
+ .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32,
.iommu_dev = data->dev,
};
- if (data->enable_4GB)
- dom->cfg.quirks |= IO_PGTABLE_QUIRK_ARM_MTK_4GB;
+ if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE))
+ dom->cfg.oas = data->enable_4GB ? 33 : 32;
+ else
+ dom->cfg.oas = 35;
dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data);
if (!dom->iop) {
@@ -314,6 +420,13 @@
/* Update our support page sizes bitmap */
dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap;
+
+update_iova_region:
+ /* Update the iova region for this domain */
+ region = data->plat_data->iova_region + domid;
+ dom->domain.geometry.aperture_start = region->iova_base;
+ dom->domain.geometry.aperture_end = region->iova_base + region->size - 1;
+ dom->domain.geometry.force_aperture = true;
return 0;
}
@@ -328,30 +441,16 @@
if (!dom)
return NULL;
- if (iommu_get_dma_cookie(&dom->domain))
- goto free_dom;
-
- if (mtk_iommu_domain_finalise(dom))
- goto put_dma_cookie;
-
- dom->domain.geometry.aperture_start = 0;
- dom->domain.geometry.aperture_end = DMA_BIT_MASK(32);
- dom->domain.geometry.force_aperture = true;
+ if (iommu_get_dma_cookie(&dom->domain)) {
+ kfree(dom);
+ return NULL;
+ }
return &dom->domain;
-
-put_dma_cookie:
- iommu_put_dma_cookie(&dom->domain);
-free_dom:
- kfree(dom);
- return NULL;
}
static void mtk_iommu_domain_free(struct iommu_domain *domain)
{
- struct mtk_iommu_domain *dom = to_mtk_domain(domain);
-
- free_io_pgtable_ops(dom->iop);
iommu_put_dma_cookie(domain);
kfree(to_mtk_domain(domain));
}
@@ -359,135 +458,171 @@
static int mtk_iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
+ struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata;
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
- struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
+ struct device *m4udev = data->dev;
+ int ret, domid;
- if (!data)
- return -ENODEV;
+ domid = mtk_iommu_get_domain_id(dev, data->plat_data);
+ if (domid < 0)
+ return domid;
- /* Update the pgtable base address register of the M4U HW */
- if (!data->m4u_dom) {
- data->m4u_dom = dom;
- writel(dom->cfg.arm_v7s_cfg.ttbr[0],
- data->base + REG_MMU_PT_BASE_ADDR);
+ if (!dom->data) {
+ /* Data is in the frstdata in sharing pgtable case. */
+ frstdata = mtk_iommu_get_m4u_data();
+
+ if (mtk_iommu_domain_finalise(dom, frstdata, domid))
+ return -ENODEV;
+ dom->data = data;
}
- mtk_iommu_config(data, dev, true);
+ mutex_lock(&data->mutex);
+ if (!data->m4u_dom) { /* Initialize the M4U HW */
+ ret = pm_runtime_resume_and_get(m4udev);
+ if (ret < 0)
+ goto err_unlock;
+
+ ret = mtk_iommu_hw_init(data);
+ if (ret) {
+ pm_runtime_put(m4udev);
+ goto err_unlock;
+ }
+ data->m4u_dom = dom;
+ writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK,
+ data->base + REG_MMU_PT_BASE_ADDR);
+
+ pm_runtime_put(m4udev);
+ }
+ mutex_unlock(&data->mutex);
+
+ mtk_iommu_config(data, dev, true, domid);
return 0;
+
+err_unlock:
+ mutex_unlock(&data->mutex);
+ return ret;
}
static void mtk_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
- struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
+ struct mtk_iommu_data *data = dev_iommu_priv_get(dev);
- if (!data)
- return;
-
- mtk_iommu_config(data, dev, false);
+ mtk_iommu_config(data, dev, false, 0);
}
static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
- phys_addr_t paddr, size_t size, int prot)
+ phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
- unsigned long flags;
- int ret;
- spin_lock_irqsave(&dom->pgtlock, flags);
- ret = dom->iop->map(dom->iop, iova, paddr & DMA_BIT_MASK(32),
- size, prot);
- spin_unlock_irqrestore(&dom->pgtlock, flags);
+ /* The "4GB mode" M4U physically can not use the lower remap of Dram. */
+ if (dom->data->enable_4GB)
+ paddr |= BIT_ULL(32);
- return ret;
+ /* Synchronize with the tlb_lock */
+ return dom->iop->map(dom->iop, iova, paddr, size, prot, gfp);
}
static size_t mtk_iommu_unmap(struct iommu_domain *domain,
- unsigned long iova, size_t size)
+ unsigned long iova, size_t size,
+ struct iommu_iotlb_gather *gather)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
- unsigned long flags;
- size_t unmapsz;
+ unsigned long end = iova + size - 1;
- spin_lock_irqsave(&dom->pgtlock, flags);
- unmapsz = dom->iop->unmap(dom->iop, iova, size);
- spin_unlock_irqrestore(&dom->pgtlock, flags);
-
- return unmapsz;
+ if (gather->start > iova)
+ gather->start = iova;
+ if (gather->end < end)
+ gather->end = end;
+ return dom->iop->unmap(dom->iop, iova, size, gather);
}
-static void mtk_iommu_iotlb_sync(struct iommu_domain *domain)
+static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain)
{
- mtk_iommu_tlb_flush_all(mtk_iommu_get_m4u_data());
+ struct mtk_iommu_domain *dom = to_mtk_domain(domain);
+
+ mtk_iommu_tlb_flush_all(dom->data);
+}
+
+static void mtk_iommu_iotlb_sync(struct iommu_domain *domain,
+ struct iommu_iotlb_gather *gather)
+{
+ struct mtk_iommu_domain *dom = to_mtk_domain(domain);
+ size_t length = gather->end - gather->start + 1;
+
+ mtk_iommu_tlb_flush_range_sync(gather->start, length, gather->pgsize,
+ dom->data);
+}
+
+static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
+ size_t size)
+{
+ struct mtk_iommu_domain *dom = to_mtk_domain(domain);
+
+ mtk_iommu_tlb_flush_range_sync(iova, size, size, dom->data);
}
static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
- struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
- unsigned long flags;
phys_addr_t pa;
- spin_lock_irqsave(&dom->pgtlock, flags);
pa = dom->iop->iova_to_phys(dom->iop, iova);
- spin_unlock_irqrestore(&dom->pgtlock, flags);
-
- if (data->enable_4GB && pa < MTK_IOMMU_4GB_MODE_REMAP_BASE)
- pa |= BIT_ULL(32);
+ if (dom->data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE)
+ pa &= ~BIT_ULL(32);
return pa;
}
-static int mtk_iommu_add_device(struct device *dev)
+static struct iommu_device *mtk_iommu_probe_device(struct device *dev)
{
+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
struct mtk_iommu_data *data;
- struct iommu_group *group;
- if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops)
- return -ENODEV; /* Not a iommu client device */
+ if (!fwspec || fwspec->ops != &mtk_iommu_ops)
+ return ERR_PTR(-ENODEV); /* Not a iommu client device */
- data = dev->iommu_fwspec->iommu_priv;
- iommu_device_link(&data->iommu, dev);
+ data = dev_iommu_priv_get(dev);
- group = iommu_group_get_for_dev(dev);
- if (IS_ERR(group))
- return PTR_ERR(group);
-
- iommu_group_put(group);
- return 0;
+ return &data->iommu;
}
-static void mtk_iommu_remove_device(struct device *dev)
+static void mtk_iommu_release_device(struct device *dev)
{
- struct mtk_iommu_data *data;
+ struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
- if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops)
+ if (!fwspec || fwspec->ops != &mtk_iommu_ops)
return;
- data = dev->iommu_fwspec->iommu_priv;
- iommu_device_unlink(&data->iommu, dev);
-
- iommu_group_remove_device(dev);
iommu_fwspec_free(dev);
}
static struct iommu_group *mtk_iommu_device_group(struct device *dev)
{
struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
+ struct iommu_group *group;
+ int domid;
if (!data)
return ERR_PTR(-ENODEV);
- /* All the client devices are in the same m4u iommu-group */
- if (!data->m4u_group) {
- data->m4u_group = iommu_group_alloc();
- if (IS_ERR(data->m4u_group))
- dev_err(dev, "Failed to allocate M4U IOMMU group\n");
+ domid = mtk_iommu_get_domain_id(dev, data->plat_data);
+ if (domid < 0)
+ return ERR_PTR(domid);
+
+ mutex_lock(&data->mutex);
+ group = data->m4u_group[domid];
+ if (!group) {
+ group = iommu_group_alloc();
+ if (!IS_ERR(group))
+ data->m4u_group[domid] = group;
} else {
- iommu_group_ref_get(data->m4u_group);
+ iommu_group_ref_get(group);
}
- return data->m4u_group;
+ mutex_unlock(&data->mutex);
+ return group;
}
static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
@@ -500,49 +635,78 @@
return -EINVAL;
}
- if (!dev->iommu_fwspec->iommu_priv) {
+ if (!dev_iommu_priv_get(dev)) {
/* Get the m4u device */
m4updev = of_find_device_by_node(args->np);
if (WARN_ON(!m4updev))
return -EINVAL;
- dev->iommu_fwspec->iommu_priv = platform_get_drvdata(m4updev);
+ dev_iommu_priv_set(dev, platform_get_drvdata(m4updev));
}
return iommu_fwspec_add_ids(dev, args->args, 1);
}
-static struct iommu_ops mtk_iommu_ops = {
+static void mtk_iommu_get_resv_regions(struct device *dev,
+ struct list_head *head)
+{
+ struct mtk_iommu_data *data = dev_iommu_priv_get(dev);
+ unsigned int domid = mtk_iommu_get_domain_id(dev, data->plat_data), i;
+ const struct mtk_iommu_iova_region *resv, *curdom;
+ struct iommu_resv_region *region;
+ int prot = IOMMU_WRITE | IOMMU_READ;
+
+ if ((int)domid < 0)
+ return;
+ curdom = data->plat_data->iova_region + domid;
+ for (i = 0; i < data->plat_data->iova_region_nr; i++) {
+ resv = data->plat_data->iova_region + i;
+
+ /* Only reserve when the region is inside the current domain */
+ if (resv->iova_base <= curdom->iova_base ||
+ resv->iova_base + resv->size >= curdom->iova_base + curdom->size)
+ continue;
+
+ region = iommu_alloc_resv_region(resv->iova_base, resv->size,
+ prot, IOMMU_RESV_RESERVED);
+ if (!region)
+ return;
+
+ list_add_tail(®ion->list, head);
+ }
+}
+
+static const struct iommu_ops mtk_iommu_ops = {
.domain_alloc = mtk_iommu_domain_alloc,
.domain_free = mtk_iommu_domain_free,
.attach_dev = mtk_iommu_attach_device,
.detach_dev = mtk_iommu_detach_device,
.map = mtk_iommu_map,
.unmap = mtk_iommu_unmap,
- .flush_iotlb_all = mtk_iommu_iotlb_sync,
+ .flush_iotlb_all = mtk_iommu_flush_iotlb_all,
.iotlb_sync = mtk_iommu_iotlb_sync,
+ .iotlb_sync_map = mtk_iommu_sync_map,
.iova_to_phys = mtk_iommu_iova_to_phys,
- .add_device = mtk_iommu_add_device,
- .remove_device = mtk_iommu_remove_device,
+ .probe_device = mtk_iommu_probe_device,
+ .release_device = mtk_iommu_release_device,
.device_group = mtk_iommu_device_group,
.of_xlate = mtk_iommu_of_xlate,
+ .get_resv_regions = mtk_iommu_get_resv_regions,
+ .put_resv_regions = generic_iommu_put_resv_regions,
.pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
};
static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
{
u32 regval;
- int ret;
- ret = clk_prepare_enable(data->bclk);
- if (ret) {
- dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret);
- return ret;
+ if (data->plat_data->m4u_plat == M4U_MT8173) {
+ regval = F_MMU_PREFETCH_RT_REPLACE_MOD |
+ F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173;
+ } else {
+ regval = readl_relaxed(data->base + REG_MMU_CTRL_REG);
+ regval |= F_MMU_TF_PROT_TO_PROGRAM_ADDR;
}
-
- regval = F_MMU_TF_PROTECT_SEL(2, data);
- if (data->m4u_plat == M4U_MT8173)
- regval |= F_MMU_PREFETCH_RT_REPLACE_MOD;
writel_relaxed(regval, data->base + REG_MMU_CTRL_REG);
regval = F_L2_MULIT_HIT_EN |
@@ -562,14 +726,15 @@
F_INT_PRETETCH_TRANSATION_FIFO_FAULT;
writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL);
- if (data->m4u_plat == M4U_MT8173)
+ if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR))
regval = (data->protect_base >> 1) | (data->enable_4GB << 31);
else
regval = lower_32_bits(data->protect_base) |
upper_32_bits(data->protect_base);
writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR);
- if (data->enable_4GB && data->m4u_plat != M4U_MT8173) {
+ if (data->enable_4GB &&
+ MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) {
/*
* If 4GB mode is enabled, the validate PA range is from
* 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30].
@@ -578,15 +743,27 @@
writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG);
}
writel_relaxed(0, data->base + REG_MMU_DCM_DIS);
+ if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) {
+ /* write command throttling mode */
+ regval = readl_relaxed(data->base + REG_MMU_WR_LEN_CTRL);
+ regval &= ~F_MMU_WR_THROT_DIS_MASK;
+ writel_relaxed(regval, data->base + REG_MMU_WR_LEN_CTRL);
+ }
- /* It's MISC control register whose default value is ok except mt8173.*/
- if (data->m4u_plat == M4U_MT8173)
- writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE);
+ if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) {
+ /* The register is called STANDARD_AXI_MODE in this case */
+ regval = 0;
+ } else {
+ regval = readl_relaxed(data->base + REG_MMU_MISC_CTRL);
+ regval &= ~F_MMU_STANDARD_AXI_MODE_MASK;
+ if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN))
+ regval &= ~F_MMU_IN_ORDER_WR_EN_MASK;
+ }
+ writel_relaxed(regval, data->base + REG_MMU_MISC_CTRL);
if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0,
dev_name(data->dev), (void *)data)) {
writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR);
- clk_disable_unprepare(data->bclk);
dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq);
return -ENODEV;
}
@@ -603,17 +780,23 @@
{
struct mtk_iommu_data *data;
struct device *dev = &pdev->dev;
+ struct device_node *larbnode, *smicomm_node;
+ struct platform_device *plarbdev;
+ struct device_link *link;
struct resource *res;
resource_size_t ioaddr;
struct component_match *match = NULL;
+ struct regmap *infracfg;
void *protect;
int i, larb_nr, ret;
+ u32 val;
+ char *p;
data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
if (!data)
return -ENOMEM;
data->dev = dev;
- data->m4u_plat = (enum mtk_iommu_plat)of_device_get_match_data(dev);
+ data->plat_data = of_device_get_match_data(dev);
/* Protect memory. HW will access here while translation fault.*/
protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL);
@@ -621,8 +804,28 @@
return -ENOMEM;
data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN);
- /* Whether the current dram is over 4GB */
- data->enable_4GB = !!(max_pfn > (BIT_ULL(32) >> PAGE_SHIFT));
+ if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) {
+ switch (data->plat_data->m4u_plat) {
+ case M4U_MT2712:
+ p = "mediatek,mt2712-infracfg";
+ break;
+ case M4U_MT8173:
+ p = "mediatek,mt8173-infracfg";
+ break;
+ default:
+ p = NULL;
+ }
+
+ infracfg = syscon_regmap_lookup_by_compatible(p);
+
+ if (IS_ERR(infracfg))
+ return PTR_ERR(infracfg);
+
+ ret = regmap_read(infracfg, REG_INFRA_MISC, &val);
+ if (ret)
+ return ret;
+ data->enable_4GB = !!(val & F_DDR_4GB_SUPPORT_EN);
+ }
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
data->base = devm_ioremap_resource(dev, res);
@@ -634,65 +837,104 @@
if (data->irq < 0)
return data->irq;
- data->bclk = devm_clk_get(dev, "bclk");
- if (IS_ERR(data->bclk))
- return PTR_ERR(data->bclk);
+ if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) {
+ data->bclk = devm_clk_get(dev, "bclk");
+ if (IS_ERR(data->bclk))
+ return PTR_ERR(data->bclk);
+ }
larb_nr = of_count_phandle_with_args(dev->of_node,
"mediatek,larbs", NULL);
if (larb_nr < 0)
return larb_nr;
- data->smi_imu.larb_nr = larb_nr;
for (i = 0; i < larb_nr; i++) {
- struct device_node *larbnode;
- struct platform_device *plarbdev;
u32 id;
larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);
if (!larbnode)
return -EINVAL;
- if (!of_device_is_available(larbnode))
+ if (!of_device_is_available(larbnode)) {
+ of_node_put(larbnode);
continue;
+ }
ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id);
if (ret)/* The id is consecutive if there is no this property */
id = i;
plarbdev = of_find_device_by_node(larbnode);
- if (!plarbdev)
+ if (!plarbdev) {
+ of_node_put(larbnode);
return -EPROBE_DEFER;
- data->smi_imu.larb_imu[id].dev = &plarbdev->dev;
+ }
+ data->larb_imu[id].dev = &plarbdev->dev;
component_match_add_release(dev, &match, release_of,
compare_of, larbnode);
}
- platform_set_drvdata(pdev, data);
+ /* Get smi-common dev from the last larb. */
+ smicomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0);
+ if (!smicomm_node)
+ return -EINVAL;
- ret = mtk_iommu_hw_init(data);
- if (ret)
- return ret;
+ plarbdev = of_find_device_by_node(smicomm_node);
+ of_node_put(smicomm_node);
+ data->smicomm_dev = &plarbdev->dev;
+
+ pm_runtime_enable(dev);
+
+ link = device_link_add(data->smicomm_dev, dev,
+ DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
+ if (!link) {
+ dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev));
+ ret = -EINVAL;
+ goto out_runtime_disable;
+ }
+
+ platform_set_drvdata(pdev, data);
+ mutex_init(&data->mutex);
ret = iommu_device_sysfs_add(&data->iommu, dev, NULL,
"mtk-iommu.%pa", &ioaddr);
if (ret)
- return ret;
+ goto out_link_remove;
iommu_device_set_ops(&data->iommu, &mtk_iommu_ops);
iommu_device_set_fwnode(&data->iommu, &pdev->dev.of_node->fwnode);
ret = iommu_device_register(&data->iommu);
if (ret)
- return ret;
+ goto out_sysfs_remove;
+ spin_lock_init(&data->tlb_lock);
list_add_tail(&data->list, &m4ulist);
- if (!iommu_present(&platform_bus_type))
- bus_set_iommu(&platform_bus_type, &mtk_iommu_ops);
+ if (!iommu_present(&platform_bus_type)) {
+ ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops);
+ if (ret)
+ goto out_list_del;
+ }
- return component_master_add_with_match(dev, &mtk_iommu_com_ops, match);
+ ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match);
+ if (ret)
+ goto out_bus_set_null;
+ return ret;
+
+out_bus_set_null:
+ bus_set_iommu(&platform_bus_type, NULL);
+out_list_del:
+ list_del(&data->list);
+ iommu_device_unregister(&data->iommu);
+out_sysfs_remove:
+ iommu_device_sysfs_remove(&data->iommu);
+out_link_remove:
+ device_link_remove(data->smicomm_dev, dev);
+out_runtime_disable:
+ pm_runtime_disable(dev);
+ return ret;
}
static int mtk_iommu_remove(struct platform_device *pdev)
@@ -702,36 +944,38 @@
iommu_device_sysfs_remove(&data->iommu);
iommu_device_unregister(&data->iommu);
- if (iommu_present(&platform_bus_type))
- bus_set_iommu(&platform_bus_type, NULL);
+ list_del(&data->list);
- clk_disable_unprepare(data->bclk);
+ device_link_remove(data->smicomm_dev, &pdev->dev);
+ pm_runtime_disable(&pdev->dev);
devm_free_irq(&pdev->dev, data->irq, data);
component_master_del(&pdev->dev, &mtk_iommu_com_ops);
return 0;
}
-static int __maybe_unused mtk_iommu_suspend(struct device *dev)
+static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev)
{
struct mtk_iommu_data *data = dev_get_drvdata(dev);
struct mtk_iommu_suspend_reg *reg = &data->reg;
void __iomem *base = data->base;
- reg->standard_axi_mode = readl_relaxed(base +
- REG_MMU_STANDARD_AXI_MODE);
+ reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL);
+ reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL);
reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS);
reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG);
reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0);
reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL);
reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR);
+ reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG);
clk_disable_unprepare(data->bclk);
return 0;
}
-static int __maybe_unused mtk_iommu_resume(struct device *dev)
+static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev)
{
struct mtk_iommu_data *data = dev_get_drvdata(dev);
struct mtk_iommu_suspend_reg *reg = &data->reg;
+ struct mtk_iommu_domain *m4u_dom = data->m4u_dom;
void __iomem *base = data->base;
int ret;
@@ -740,26 +984,96 @@
dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret);
return ret;
}
- writel_relaxed(reg->standard_axi_mode,
- base + REG_MMU_STANDARD_AXI_MODE);
+
+ /*
+ * Uppon first resume, only enable the clk and return, since the values of the
+ * registers are not yet set.
+ */
+ if (!m4u_dom)
+ return 0;
+
+ writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL);
+ writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL);
writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS);
writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG);
writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0);
writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL);
writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR);
- if (data->m4u_dom)
- writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
- base + REG_MMU_PT_BASE_ADDR);
+ writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG);
+ writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, base + REG_MMU_PT_BASE_ADDR);
return 0;
}
static const struct dev_pm_ops mtk_iommu_pm_ops = {
- SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
+ SET_RUNTIME_PM_OPS(mtk_iommu_runtime_suspend, mtk_iommu_runtime_resume, NULL)
+ SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+ pm_runtime_force_resume)
+};
+
+static const struct mtk_iommu_plat_data mt2712_data = {
+ .m4u_plat = M4U_MT2712,
+ .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG,
+ .inv_sel_reg = REG_MMU_INV_SEL_GEN1,
+ .iova_region = single_domain,
+ .iova_region_nr = ARRAY_SIZE(single_domain),
+ .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}},
+};
+
+static const struct mtk_iommu_plat_data mt6779_data = {
+ .m4u_plat = M4U_MT6779,
+ .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN,
+ .inv_sel_reg = REG_MMU_INV_SEL_GEN2,
+ .iova_region = single_domain,
+ .iova_region_nr = ARRAY_SIZE(single_domain),
+ .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}},
+};
+
+static const struct mtk_iommu_plat_data mt8167_data = {
+ .m4u_plat = M4U_MT8167,
+ .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR,
+ .inv_sel_reg = REG_MMU_INV_SEL_GEN1,
+ .iova_region = single_domain,
+ .iova_region_nr = ARRAY_SIZE(single_domain),
+ .larbid_remap = {{0}, {1}, {2}}, /* Linear mapping. */
+};
+
+static const struct mtk_iommu_plat_data mt8173_data = {
+ .m4u_plat = M4U_MT8173,
+ .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI |
+ HAS_LEGACY_IVRP_PADDR,
+ .inv_sel_reg = REG_MMU_INV_SEL_GEN1,
+ .iova_region = single_domain,
+ .iova_region_nr = ARRAY_SIZE(single_domain),
+ .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */
+};
+
+static const struct mtk_iommu_plat_data mt8183_data = {
+ .m4u_plat = M4U_MT8183,
+ .flags = RESET_AXI,
+ .inv_sel_reg = REG_MMU_INV_SEL_GEN1,
+ .iova_region = single_domain,
+ .iova_region_nr = ARRAY_SIZE(single_domain),
+ .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}},
+};
+
+static const struct mtk_iommu_plat_data mt8192_data = {
+ .m4u_plat = M4U_MT8192,
+ .flags = HAS_BCLK | HAS_SUB_COMM | OUT_ORDER_WR_EN |
+ WR_THROT_EN | IOVA_34_EN,
+ .inv_sel_reg = REG_MMU_INV_SEL_GEN2,
+ .iova_region = mt8192_multi_dom,
+ .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom),
+ .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20},
+ {0, 14, 16}, {0, 13, 18, 17}},
};
static const struct of_device_id mtk_iommu_of_ids[] = {
- { .compatible = "mediatek,mt2712-m4u", .data = (void *)M4U_MT2712},
- { .compatible = "mediatek,mt8173-m4u", .data = (void *)M4U_MT8173},
+ { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data},
+ { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data},
+ { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data},
+ { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data},
+ { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data},
+ { .compatible = "mediatek,mt8192-m4u", .data = &mt8192_data},
{}
};
@@ -768,7 +1082,7 @@
.remove = mtk_iommu_remove,
.driver = {
.name = "mtk-iommu",
- .of_match_table = of_match_ptr(mtk_iommu_of_ids),
+ .of_match_table = mtk_iommu_of_ids,
.pm = &mtk_iommu_pm_ops,
}
};
--
Gitblit v1.6.2