// SPDX-License-Identifier: GPL-2.0
|
/*
|
* Copyright (C) 2018 Intel Corporation.
|
* Copyright 2018 Google LLC.
|
*
|
* Author: Tuukka Toivonen <tuukka.toivonen@intel.com>
|
* Author: Sakari Ailus <sakari.ailus@linux.intel.com>
|
* Author: Samu Onkalo <samu.onkalo@intel.com>
|
* Author: Tomasz Figa <tfiga@chromium.org>
|
*
|
*/
|
|
#include <linux/dma-mapping.h>
|
#include <linux/iopoll.h>
|
#include <linux/pm_runtime.h>
|
#include <linux/slab.h>
|
#include <linux/vmalloc.h>
|
|
#include <asm/set_memory.h>
|
|
#include "ipu3-mmu.h"
|
|
#define IPU3_PT_BITS 10
|
#define IPU3_PT_PTES (1UL << IPU3_PT_BITS)
|
#define IPU3_PT_SIZE (IPU3_PT_PTES << 2)
|
#define IPU3_PT_ORDER (IPU3_PT_SIZE >> PAGE_SHIFT)
|
|
#define IPU3_ADDR2PTE(addr) ((addr) >> IPU3_PAGE_SHIFT)
|
#define IPU3_PTE2ADDR(pte) ((phys_addr_t)(pte) << IPU3_PAGE_SHIFT)
|
|
#define IPU3_L2PT_SHIFT IPU3_PT_BITS
|
#define IPU3_L2PT_MASK ((1UL << IPU3_L2PT_SHIFT) - 1)
|
|
#define IPU3_L1PT_SHIFT IPU3_PT_BITS
|
#define IPU3_L1PT_MASK ((1UL << IPU3_L1PT_SHIFT) - 1)
|
|
#define IPU3_MMU_ADDRESS_BITS (IPU3_PAGE_SHIFT + \
|
IPU3_L2PT_SHIFT + \
|
IPU3_L1PT_SHIFT)
|
|
#define IMGU_REG_BASE 0x4000
|
#define REG_TLB_INVALIDATE (IMGU_REG_BASE + 0x300)
|
#define TLB_INVALIDATE 1
|
#define REG_L1_PHYS (IMGU_REG_BASE + 0x304) /* 27-bit pfn */
|
#define REG_GP_HALT (IMGU_REG_BASE + 0x5dc)
|
#define REG_GP_HALTED (IMGU_REG_BASE + 0x5e0)
|
|
struct imgu_mmu {
|
struct device *dev;
|
void __iomem *base;
|
/* protect access to l2pts, l1pt */
|
spinlock_t lock;
|
|
void *dummy_page;
|
u32 dummy_page_pteval;
|
|
u32 *dummy_l2pt;
|
u32 dummy_l2pt_pteval;
|
|
u32 **l2pts;
|
u32 *l1pt;
|
|
struct imgu_mmu_info geometry;
|
};
|
|
static inline struct imgu_mmu *to_imgu_mmu(struct imgu_mmu_info *info)
|
{
|
return container_of(info, struct imgu_mmu, geometry);
|
}
|
|
/**
|
* imgu_mmu_tlb_invalidate - invalidate translation look-aside buffer
|
* @mmu: MMU to perform the invalidate operation on
|
*
|
* This function invalidates the whole TLB. Must be called when the hardware
|
* is powered on.
|
*/
|
static void imgu_mmu_tlb_invalidate(struct imgu_mmu *mmu)
|
{
|
writel(TLB_INVALIDATE, mmu->base + REG_TLB_INVALIDATE);
|
}
|
|
static void call_if_imgu_is_powered(struct imgu_mmu *mmu,
|
void (*func)(struct imgu_mmu *mmu))
|
{
|
if (!pm_runtime_get_if_in_use(mmu->dev))
|
return;
|
|
func(mmu);
|
pm_runtime_put(mmu->dev);
|
}
|
|
/**
|
* imgu_mmu_set_halt - set CIO gate halt bit
|
* @mmu: MMU to set the CIO gate bit in.
|
* @halt: Desired state of the gate bit.
|
*
|
* This function sets the CIO gate bit that controls whether external memory
|
* accesses are allowed. Must be called when the hardware is powered on.
|
*/
|
static void imgu_mmu_set_halt(struct imgu_mmu *mmu, bool halt)
|
{
|
int ret;
|
u32 val;
|
|
writel(halt, mmu->base + REG_GP_HALT);
|
ret = readl_poll_timeout(mmu->base + REG_GP_HALTED,
|
val, (val & 1) == halt, 1000, 100000);
|
|
if (ret)
|
dev_err(mmu->dev, "failed to %s CIO gate halt\n",
|
halt ? "set" : "clear");
|
}
|
|
/**
|
* imgu_mmu_alloc_page_table - allocate a pre-filled page table
|
* @pteval: Value to initialize for page table entries with.
|
*
|
* Return: Pointer to allocated page table or NULL on failure.
|
*/
|
static u32 *imgu_mmu_alloc_page_table(u32 pteval)
|
{
|
u32 *pt;
|
int pte;
|
|
pt = (u32 *)__get_free_page(GFP_KERNEL);
|
if (!pt)
|
return NULL;
|
|
for (pte = 0; pte < IPU3_PT_PTES; pte++)
|
pt[pte] = pteval;
|
|
set_memory_uc((unsigned long)pt, IPU3_PT_ORDER);
|
|
return pt;
|
}
|
|
/**
|
* imgu_mmu_free_page_table - free page table
|
* @pt: Page table to free.
|
*/
|
static void imgu_mmu_free_page_table(u32 *pt)
|
{
|
set_memory_wb((unsigned long)pt, IPU3_PT_ORDER);
|
free_page((unsigned long)pt);
|
}
|
|
/**
|
* address_to_pte_idx - split IOVA into L1 and L2 page table indices
|
* @iova: IOVA to split.
|
* @l1pt_idx: Output for the L1 page table index.
|
* @l2pt_idx: Output for the L2 page index.
|
*/
|
static inline void address_to_pte_idx(unsigned long iova, u32 *l1pt_idx,
|
u32 *l2pt_idx)
|
{
|
iova >>= IPU3_PAGE_SHIFT;
|
|
if (l2pt_idx)
|
*l2pt_idx = iova & IPU3_L2PT_MASK;
|
|
iova >>= IPU3_L2PT_SHIFT;
|
|
if (l1pt_idx)
|
*l1pt_idx = iova & IPU3_L1PT_MASK;
|
}
|
|
static u32 *imgu_mmu_get_l2pt(struct imgu_mmu *mmu, u32 l1pt_idx)
|
{
|
unsigned long flags;
|
u32 *l2pt, *new_l2pt;
|
u32 pteval;
|
|
spin_lock_irqsave(&mmu->lock, flags);
|
|
l2pt = mmu->l2pts[l1pt_idx];
|
if (l2pt) {
|
spin_unlock_irqrestore(&mmu->lock, flags);
|
return l2pt;
|
}
|
|
spin_unlock_irqrestore(&mmu->lock, flags);
|
|
new_l2pt = imgu_mmu_alloc_page_table(mmu->dummy_page_pteval);
|
if (!new_l2pt)
|
return NULL;
|
|
spin_lock_irqsave(&mmu->lock, flags);
|
|
dev_dbg(mmu->dev, "allocated page table %p for l1pt_idx %u\n",
|
new_l2pt, l1pt_idx);
|
|
l2pt = mmu->l2pts[l1pt_idx];
|
if (l2pt) {
|
spin_unlock_irqrestore(&mmu->lock, flags);
|
imgu_mmu_free_page_table(new_l2pt);
|
return l2pt;
|
}
|
|
l2pt = new_l2pt;
|
mmu->l2pts[l1pt_idx] = new_l2pt;
|
|
pteval = IPU3_ADDR2PTE(virt_to_phys(new_l2pt));
|
mmu->l1pt[l1pt_idx] = pteval;
|
|
spin_unlock_irqrestore(&mmu->lock, flags);
|
return l2pt;
|
}
|
|
static int __imgu_mmu_map(struct imgu_mmu *mmu, unsigned long iova,
|
phys_addr_t paddr)
|
{
|
u32 l1pt_idx, l2pt_idx;
|
unsigned long flags;
|
u32 *l2pt;
|
|
if (!mmu)
|
return -ENODEV;
|
|
address_to_pte_idx(iova, &l1pt_idx, &l2pt_idx);
|
|
l2pt = imgu_mmu_get_l2pt(mmu, l1pt_idx);
|
if (!l2pt)
|
return -ENOMEM;
|
|
spin_lock_irqsave(&mmu->lock, flags);
|
|
if (l2pt[l2pt_idx] != mmu->dummy_page_pteval) {
|
spin_unlock_irqrestore(&mmu->lock, flags);
|
return -EBUSY;
|
}
|
|
l2pt[l2pt_idx] = IPU3_ADDR2PTE(paddr);
|
|
spin_unlock_irqrestore(&mmu->lock, flags);
|
|
return 0;
|
}
|
|
/**
|
* imgu_mmu_map - map a buffer to a physical address
|
*
|
* @info: MMU mappable range
|
* @iova: the virtual address
|
* @paddr: the physical address
|
* @size: length of the mappable area
|
*
|
* The function has been adapted from iommu_map() in
|
* drivers/iommu/iommu.c .
|
*/
|
int imgu_mmu_map(struct imgu_mmu_info *info, unsigned long iova,
|
phys_addr_t paddr, size_t size)
|
{
|
struct imgu_mmu *mmu = to_imgu_mmu(info);
|
int ret = 0;
|
|
/*
|
* both the virtual address and the physical one, as well as
|
* the size of the mapping, must be aligned (at least) to the
|
* size of the smallest page supported by the hardware
|
*/
|
if (!IS_ALIGNED(iova | paddr | size, IPU3_PAGE_SIZE)) {
|
dev_err(mmu->dev, "unaligned: iova 0x%lx pa %pa size 0x%zx\n",
|
iova, &paddr, size);
|
return -EINVAL;
|
}
|
|
dev_dbg(mmu->dev, "map: iova 0x%lx pa %pa size 0x%zx\n",
|
iova, &paddr, size);
|
|
while (size) {
|
dev_dbg(mmu->dev, "mapping: iova 0x%lx pa %pa\n", iova, &paddr);
|
|
ret = __imgu_mmu_map(mmu, iova, paddr);
|
if (ret)
|
break;
|
|
iova += IPU3_PAGE_SIZE;
|
paddr += IPU3_PAGE_SIZE;
|
size -= IPU3_PAGE_SIZE;
|
}
|
|
call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
|
|
return ret;
|
}
|
|
/**
|
* imgu_mmu_map_sg - Map a scatterlist
|
*
|
* @info: MMU mappable range
|
* @iova: the virtual address
|
* @sg: the scatterlist to map
|
* @nents: number of entries in the scatterlist
|
*
|
* The function has been adapted from default_iommu_map_sg() in
|
* drivers/iommu/iommu.c .
|
*/
|
size_t imgu_mmu_map_sg(struct imgu_mmu_info *info, unsigned long iova,
|
struct scatterlist *sg, unsigned int nents)
|
{
|
struct imgu_mmu *mmu = to_imgu_mmu(info);
|
struct scatterlist *s;
|
size_t s_length, mapped = 0;
|
unsigned int i;
|
int ret;
|
|
for_each_sg(sg, s, nents, i) {
|
phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset;
|
|
s_length = s->length;
|
|
if (!IS_ALIGNED(s->offset, IPU3_PAGE_SIZE))
|
goto out_err;
|
|
/* must be IPU3_PAGE_SIZE aligned to be mapped singlely */
|
if (i == nents - 1 && !IS_ALIGNED(s->length, IPU3_PAGE_SIZE))
|
s_length = PAGE_ALIGN(s->length);
|
|
ret = imgu_mmu_map(info, iova + mapped, phys, s_length);
|
if (ret)
|
goto out_err;
|
|
mapped += s_length;
|
}
|
|
call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
|
|
return mapped;
|
|
out_err:
|
/* undo mappings already done */
|
imgu_mmu_unmap(info, iova, mapped);
|
|
return 0;
|
}
|
|
static size_t __imgu_mmu_unmap(struct imgu_mmu *mmu,
|
unsigned long iova, size_t size)
|
{
|
u32 l1pt_idx, l2pt_idx;
|
unsigned long flags;
|
size_t unmap = size;
|
u32 *l2pt;
|
|
if (!mmu)
|
return 0;
|
|
address_to_pte_idx(iova, &l1pt_idx, &l2pt_idx);
|
|
spin_lock_irqsave(&mmu->lock, flags);
|
|
l2pt = mmu->l2pts[l1pt_idx];
|
if (!l2pt) {
|
spin_unlock_irqrestore(&mmu->lock, flags);
|
return 0;
|
}
|
|
if (l2pt[l2pt_idx] == mmu->dummy_page_pteval)
|
unmap = 0;
|
|
l2pt[l2pt_idx] = mmu->dummy_page_pteval;
|
|
spin_unlock_irqrestore(&mmu->lock, flags);
|
|
return unmap;
|
}
|
|
/**
|
* imgu_mmu_unmap - Unmap a buffer
|
*
|
* @info: MMU mappable range
|
* @iova: the virtual address
|
* @size: the length of the buffer
|
*
|
* The function has been adapted from iommu_unmap() in
|
* drivers/iommu/iommu.c .
|
*/
|
size_t imgu_mmu_unmap(struct imgu_mmu_info *info, unsigned long iova,
|
size_t size)
|
{
|
struct imgu_mmu *mmu = to_imgu_mmu(info);
|
size_t unmapped_page, unmapped = 0;
|
|
/*
|
* The virtual address, as well as the size of the mapping, must be
|
* aligned (at least) to the size of the smallest page supported
|
* by the hardware
|
*/
|
if (!IS_ALIGNED(iova | size, IPU3_PAGE_SIZE)) {
|
dev_err(mmu->dev, "unaligned: iova 0x%lx size 0x%zx\n",
|
iova, size);
|
return -EINVAL;
|
}
|
|
dev_dbg(mmu->dev, "unmap this: iova 0x%lx size 0x%zx\n", iova, size);
|
|
/*
|
* Keep iterating until we either unmap 'size' bytes (or more)
|
* or we hit an area that isn't mapped.
|
*/
|
while (unmapped < size) {
|
unmapped_page = __imgu_mmu_unmap(mmu, iova, IPU3_PAGE_SIZE);
|
if (!unmapped_page)
|
break;
|
|
dev_dbg(mmu->dev, "unmapped: iova 0x%lx size 0x%zx\n",
|
iova, unmapped_page);
|
|
iova += unmapped_page;
|
unmapped += unmapped_page;
|
}
|
|
call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
|
|
return unmapped;
|
}
|
|
/**
|
* imgu_mmu_init() - initialize IPU3 MMU block
|
*
|
* @parent: struct device parent
|
* @base: IOMEM base of hardware registers.
|
*
|
* Return: Pointer to IPU3 MMU private data pointer or ERR_PTR() on error.
|
*/
|
struct imgu_mmu_info *imgu_mmu_init(struct device *parent, void __iomem *base)
|
{
|
struct imgu_mmu *mmu;
|
u32 pteval;
|
|
mmu = kzalloc(sizeof(*mmu), GFP_KERNEL);
|
if (!mmu)
|
return ERR_PTR(-ENOMEM);
|
|
mmu->dev = parent;
|
mmu->base = base;
|
spin_lock_init(&mmu->lock);
|
|
/* Disallow external memory access when having no valid page tables. */
|
imgu_mmu_set_halt(mmu, true);
|
|
/*
|
* The MMU does not have a "valid" bit, so we have to use a dummy
|
* page for invalid entries.
|
*/
|
mmu->dummy_page = (void *)__get_free_page(GFP_KERNEL);
|
if (!mmu->dummy_page)
|
goto fail_group;
|
pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_page));
|
mmu->dummy_page_pteval = pteval;
|
|
/*
|
* Allocate a dummy L2 page table with all entries pointing to
|
* the dummy page.
|
*/
|
mmu->dummy_l2pt = imgu_mmu_alloc_page_table(pteval);
|
if (!mmu->dummy_l2pt)
|
goto fail_dummy_page;
|
pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_l2pt));
|
mmu->dummy_l2pt_pteval = pteval;
|
|
/*
|
* Allocate the array of L2PT CPU pointers, initialized to zero,
|
* which means the dummy L2PT allocated above.
|
*/
|
mmu->l2pts = vzalloc(IPU3_PT_PTES * sizeof(*mmu->l2pts));
|
if (!mmu->l2pts)
|
goto fail_l2pt;
|
|
/* Allocate the L1 page table. */
|
mmu->l1pt = imgu_mmu_alloc_page_table(mmu->dummy_l2pt_pteval);
|
if (!mmu->l1pt)
|
goto fail_l2pts;
|
|
pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt));
|
writel(pteval, mmu->base + REG_L1_PHYS);
|
imgu_mmu_tlb_invalidate(mmu);
|
imgu_mmu_set_halt(mmu, false);
|
|
mmu->geometry.aperture_start = 0;
|
mmu->geometry.aperture_end = DMA_BIT_MASK(IPU3_MMU_ADDRESS_BITS);
|
|
return &mmu->geometry;
|
|
fail_l2pts:
|
vfree(mmu->l2pts);
|
fail_l2pt:
|
imgu_mmu_free_page_table(mmu->dummy_l2pt);
|
fail_dummy_page:
|
free_page((unsigned long)mmu->dummy_page);
|
fail_group:
|
kfree(mmu);
|
|
return ERR_PTR(-ENOMEM);
|
}
|
|
/**
|
* imgu_mmu_exit() - clean up IPU3 MMU block
|
*
|
* @info: MMU mappable range
|
*/
|
void imgu_mmu_exit(struct imgu_mmu_info *info)
|
{
|
struct imgu_mmu *mmu = to_imgu_mmu(info);
|
|
/* We are going to free our page tables, no more memory access. */
|
imgu_mmu_set_halt(mmu, true);
|
imgu_mmu_tlb_invalidate(mmu);
|
|
imgu_mmu_free_page_table(mmu->l1pt);
|
vfree(mmu->l2pts);
|
imgu_mmu_free_page_table(mmu->dummy_l2pt);
|
free_page((unsigned long)mmu->dummy_page);
|
kfree(mmu);
|
}
|
|
void imgu_mmu_suspend(struct imgu_mmu_info *info)
|
{
|
struct imgu_mmu *mmu = to_imgu_mmu(info);
|
|
imgu_mmu_set_halt(mmu, true);
|
}
|
|
void imgu_mmu_resume(struct imgu_mmu_info *info)
|
{
|
struct imgu_mmu *mmu = to_imgu_mmu(info);
|
u32 pteval;
|
|
imgu_mmu_set_halt(mmu, true);
|
|
pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt));
|
writel(pteval, mmu->base + REG_L1_PHYS);
|
|
imgu_mmu_tlb_invalidate(mmu);
|
imgu_mmu_set_halt(mmu, false);
|
}
|
