/*
|
* Copyright (C) 2011-2017 ARM Limited. All rights reserved.
|
*
|
* This program is free software and is provided to you under the terms of the GNU General Public License version 2
|
* as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
|
*
|
* A copy of the licence is included with the program, and can also be obtained from Free Software
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
*/
|
|
#include "mali_kernel_common.h"
|
#include "mali_osk.h"
|
#include "mali_ukk.h"
|
#include "mali_uk_types.h"
|
#include "mali_mmu_page_directory.h"
|
#include "mali_memory.h"
|
#include "mali_l2_cache.h"
|
|
static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data);
|
|
u32 mali_allocate_empty_page(mali_io_address *virt_addr)
|
{
|
_mali_osk_errcode_t err;
|
mali_io_address mapping;
|
mali_dma_addr address;
|
|
if (_MALI_OSK_ERR_OK != mali_mmu_get_table_page(&address, &mapping)) {
|
/* Allocation failed */
|
MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to get table page for empty pgdir\n"));
|
return 0;
|
}
|
|
MALI_DEBUG_ASSERT_POINTER(mapping);
|
|
err = fill_page(mapping, 0);
|
if (_MALI_OSK_ERR_OK != err) {
|
mali_mmu_release_table_page(address, mapping);
|
MALI_DEBUG_PRINT(2, ("Mali MMU: Failed to zero page\n"));
|
return 0;
|
}
|
|
*virt_addr = mapping;
|
return address;
|
}
|
|
void mali_free_empty_page(mali_dma_addr address, mali_io_address virt_addr)
|
{
|
if (MALI_INVALID_PAGE != address) {
|
mali_mmu_release_table_page(address, virt_addr);
|
}
|
}
|
|
_mali_osk_errcode_t mali_create_fault_flush_pages(mali_dma_addr *page_directory,
|
mali_io_address *page_directory_mapping,
|
mali_dma_addr *page_table, mali_io_address *page_table_mapping,
|
mali_dma_addr *data_page, mali_io_address *data_page_mapping)
|
{
|
_mali_osk_errcode_t err;
|
|
err = mali_mmu_get_table_page(data_page, data_page_mapping);
|
if (_MALI_OSK_ERR_OK == err) {
|
err = mali_mmu_get_table_page(page_table, page_table_mapping);
|
if (_MALI_OSK_ERR_OK == err) {
|
err = mali_mmu_get_table_page(page_directory, page_directory_mapping);
|
if (_MALI_OSK_ERR_OK == err) {
|
fill_page(*data_page_mapping, 0);
|
fill_page(*page_table_mapping, *data_page | MALI_MMU_FLAGS_DEFAULT);
|
fill_page(*page_directory_mapping, *page_table | MALI_MMU_FLAGS_PRESENT);
|
MALI_SUCCESS;
|
}
|
mali_mmu_release_table_page(*page_table, *page_table_mapping);
|
*page_table = MALI_INVALID_PAGE;
|
}
|
mali_mmu_release_table_page(*data_page, *data_page_mapping);
|
*data_page = MALI_INVALID_PAGE;
|
}
|
return err;
|
}
|
|
void mali_destroy_fault_flush_pages(
|
mali_dma_addr *page_directory, mali_io_address *page_directory_mapping,
|
mali_dma_addr *page_table, mali_io_address *page_table_mapping,
|
mali_dma_addr *data_page, mali_io_address *data_page_mapping)
|
{
|
if (MALI_INVALID_PAGE != *page_directory) {
|
mali_mmu_release_table_page(*page_directory, *page_directory_mapping);
|
*page_directory = MALI_INVALID_PAGE;
|
*page_directory_mapping = NULL;
|
}
|
|
if (MALI_INVALID_PAGE != *page_table) {
|
mali_mmu_release_table_page(*page_table, *page_table_mapping);
|
*page_table = MALI_INVALID_PAGE;
|
*page_table_mapping = NULL;
|
}
|
|
if (MALI_INVALID_PAGE != *data_page) {
|
mali_mmu_release_table_page(*data_page, *data_page_mapping);
|
*data_page = MALI_INVALID_PAGE;
|
*data_page_mapping = NULL;
|
}
|
}
|
|
static _mali_osk_errcode_t fill_page(mali_io_address mapping, u32 data)
|
{
|
int i;
|
MALI_DEBUG_ASSERT_POINTER(mapping);
|
|
for (i = 0; i < MALI_MMU_PAGE_SIZE / 4; i++) {
|
_mali_osk_mem_iowrite32_relaxed(mapping, i * sizeof(u32), data);
|
}
|
_mali_osk_mem_barrier();
|
MALI_SUCCESS;
|
}
|
|
_mali_osk_errcode_t mali_mmu_pagedir_map(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
|
{
|
const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
|
const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
|
_mali_osk_errcode_t err;
|
mali_io_address pde_mapping;
|
mali_dma_addr pde_phys;
|
int i, page_count;
|
u32 start_address;
|
if (last_pde < first_pde)
|
return _MALI_OSK_ERR_INVALID_ARGS;
|
|
for (i = first_pde; i <= last_pde; i++) {
|
if (0 == (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
|
i * sizeof(u32)) & MALI_MMU_FLAGS_PRESENT)) {
|
/* Page table not present */
|
MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
|
MALI_DEBUG_ASSERT(NULL == pagedir->page_entries_mapped[i]);
|
|
err = mali_mmu_get_table_page(&pde_phys, &pde_mapping);
|
if (_MALI_OSK_ERR_OK != err) {
|
MALI_PRINT_ERROR(("Failed to allocate page table page.\n"));
|
return err;
|
}
|
pagedir->page_entries_mapped[i] = pde_mapping;
|
|
/* Update PDE, mark as present */
|
_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32),
|
pde_phys | MALI_MMU_FLAGS_PRESENT);
|
|
MALI_DEBUG_ASSERT(0 == pagedir->page_entries_usage_count[i]);
|
}
|
|
if (first_pde == last_pde) {
|
pagedir->page_entries_usage_count[i] += size / MALI_MMU_PAGE_SIZE;
|
} else if (i == first_pde) {
|
start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE;
|
page_count = (start_address + MALI_MMU_VIRTUAL_PAGE_SIZE - mali_address) / MALI_MMU_PAGE_SIZE;
|
pagedir->page_entries_usage_count[i] += page_count;
|
} else if (i == last_pde) {
|
start_address = i * MALI_MMU_VIRTUAL_PAGE_SIZE;
|
page_count = (mali_address + size - start_address) / MALI_MMU_PAGE_SIZE;
|
pagedir->page_entries_usage_count[i] += page_count;
|
} else {
|
pagedir->page_entries_usage_count[i] = 1024;
|
}
|
}
|
_mali_osk_write_mem_barrier();
|
|
return _MALI_OSK_ERR_OK;
|
}
|
|
MALI_STATIC_INLINE void mali_mmu_zero_pte(mali_io_address page_table, u32 mali_address, u32 size)
|
{
|
int i;
|
const int first_pte = MALI_MMU_PTE_ENTRY(mali_address);
|
const int last_pte = MALI_MMU_PTE_ENTRY(mali_address + size - 1);
|
|
for (i = first_pte; i <= last_pte; i++) {
|
_mali_osk_mem_iowrite32_relaxed(page_table, i * sizeof(u32), 0);
|
}
|
}
|
|
static u32 mali_page_directory_get_phys_address(struct mali_page_directory *pagedir, u32 index)
|
{
|
return (_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
|
index * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK);
|
}
|
|
|
_mali_osk_errcode_t mali_mmu_pagedir_unmap(struct mali_page_directory *pagedir, u32 mali_address, u32 size)
|
{
|
const int first_pde = MALI_MMU_PDE_ENTRY(mali_address);
|
const int last_pde = MALI_MMU_PDE_ENTRY(mali_address + size - 1);
|
u32 left = size;
|
int i;
|
mali_bool pd_changed = MALI_FALSE;
|
u32 pages_to_invalidate[3]; /* hard-coded to 3: max two pages from the PT level plus max one page from PD level */
|
u32 num_pages_inv = 0;
|
mali_bool invalidate_all = MALI_FALSE; /* safety mechanism in case page_entries_usage_count is unreliable */
|
|
/* For all page directory entries in range. */
|
for (i = first_pde; i <= last_pde; i++) {
|
u32 size_in_pde, offset;
|
|
MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[i]);
|
MALI_DEBUG_ASSERT(0 != pagedir->page_entries_usage_count[i]);
|
|
/* Offset into page table, 0 if mali_address is 4MiB aligned */
|
offset = (mali_address & (MALI_MMU_VIRTUAL_PAGE_SIZE - 1));
|
if (left < MALI_MMU_VIRTUAL_PAGE_SIZE - offset) {
|
size_in_pde = left;
|
} else {
|
size_in_pde = MALI_MMU_VIRTUAL_PAGE_SIZE - offset;
|
}
|
|
pagedir->page_entries_usage_count[i] -= size_in_pde / MALI_MMU_PAGE_SIZE;
|
|
/* If entire page table is unused, free it */
|
if (0 == pagedir->page_entries_usage_count[i]) {
|
u32 page_phys;
|
void *page_virt;
|
MALI_DEBUG_PRINT(4, ("Releasing page table as this is the last reference\n"));
|
/* last reference removed, no need to zero out each PTE */
|
|
page_phys = MALI_MMU_ENTRY_ADDRESS(_mali_osk_mem_ioread32(pagedir->page_directory_mapped, i * sizeof(u32)));
|
page_virt = pagedir->page_entries_mapped[i];
|
pagedir->page_entries_mapped[i] = NULL;
|
_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);
|
|
mali_mmu_release_table_page(page_phys, page_virt);
|
pd_changed = MALI_TRUE;
|
} else {
|
MALI_DEBUG_ASSERT(num_pages_inv < 2);
|
if (num_pages_inv < 2) {
|
pages_to_invalidate[num_pages_inv] = mali_page_directory_get_phys_address(pagedir, i);
|
num_pages_inv++;
|
} else {
|
invalidate_all = MALI_TRUE;
|
}
|
|
/* If part of the page table is still in use, zero the relevant PTEs */
|
mali_mmu_zero_pte(pagedir->page_entries_mapped[i], mali_address, size_in_pde);
|
}
|
|
left -= size_in_pde;
|
mali_address += size_in_pde;
|
}
|
_mali_osk_write_mem_barrier();
|
|
/* L2 pages invalidation */
|
if (MALI_TRUE == pd_changed) {
|
MALI_DEBUG_ASSERT(num_pages_inv < 3);
|
if (num_pages_inv < 3) {
|
pages_to_invalidate[num_pages_inv] = pagedir->page_directory;
|
num_pages_inv++;
|
} else {
|
invalidate_all = MALI_TRUE;
|
}
|
}
|
|
if (invalidate_all) {
|
mali_l2_cache_invalidate_all();
|
} else {
|
mali_l2_cache_invalidate_all_pages(pages_to_invalidate, num_pages_inv);
|
}
|
|
MALI_SUCCESS;
|
}
|
|
struct mali_page_directory *mali_mmu_pagedir_alloc(void)
|
{
|
struct mali_page_directory *pagedir;
|
_mali_osk_errcode_t err;
|
mali_dma_addr phys;
|
|
pagedir = _mali_osk_calloc(1, sizeof(struct mali_page_directory));
|
if (NULL == pagedir) {
|
return NULL;
|
}
|
|
err = mali_mmu_get_table_page(&phys, &pagedir->page_directory_mapped);
|
if (_MALI_OSK_ERR_OK != err) {
|
_mali_osk_free(pagedir);
|
return NULL;
|
}
|
|
pagedir->page_directory = (u32)phys;
|
|
/* Zero page directory */
|
fill_page(pagedir->page_directory_mapped, 0);
|
|
return pagedir;
|
}
|
|
void mali_mmu_pagedir_free(struct mali_page_directory *pagedir)
|
{
|
const int num_page_table_entries = sizeof(pagedir->page_entries_mapped) / sizeof(pagedir->page_entries_mapped[0]);
|
int i;
|
|
/* Free referenced page tables and zero PDEs. */
|
for (i = 0; i < num_page_table_entries; i++) {
|
if (pagedir->page_directory_mapped && (_mali_osk_mem_ioread32(
|
pagedir->page_directory_mapped,
|
sizeof(u32)*i) & MALI_MMU_FLAGS_PRESENT)) {
|
mali_dma_addr phys = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
|
i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK;
|
_mali_osk_mem_iowrite32_relaxed(pagedir->page_directory_mapped, i * sizeof(u32), 0);
|
mali_mmu_release_table_page(phys, pagedir->page_entries_mapped[i]);
|
}
|
}
|
_mali_osk_write_mem_barrier();
|
|
/* Free the page directory page. */
|
mali_mmu_release_table_page(pagedir->page_directory, pagedir->page_directory_mapped);
|
|
_mali_osk_free(pagedir);
|
}
|
|
|
void mali_mmu_pagedir_update(struct mali_page_directory *pagedir, u32 mali_address,
|
mali_dma_addr phys_address, u32 size, u32 permission_bits)
|
{
|
u32 end_address = mali_address + size;
|
u32 mali_phys = (u32)phys_address;
|
|
/* Map physical pages into MMU page tables */
|
for (; mali_address < end_address; mali_address += MALI_MMU_PAGE_SIZE, mali_phys += MALI_MMU_PAGE_SIZE) {
|
MALI_DEBUG_ASSERT_POINTER(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)]);
|
_mali_osk_mem_iowrite32_relaxed(pagedir->page_entries_mapped[MALI_MMU_PDE_ENTRY(mali_address)],
|
MALI_MMU_PTE_ENTRY(mali_address) * sizeof(u32),
|
mali_phys | permission_bits);
|
}
|
}
|
|
void mali_mmu_pagedir_diag(struct mali_page_directory *pagedir, u32 fault_addr)
|
{
|
#if defined(DEBUG)
|
u32 pde_index, pte_index;
|
u32 pde, pte;
|
|
pde_index = MALI_MMU_PDE_ENTRY(fault_addr);
|
pte_index = MALI_MMU_PTE_ENTRY(fault_addr);
|
|
|
pde = _mali_osk_mem_ioread32(pagedir->page_directory_mapped,
|
pde_index * sizeof(u32));
|
|
|
if (pde & MALI_MMU_FLAGS_PRESENT) {
|
u32 pte_addr = MALI_MMU_ENTRY_ADDRESS(pde);
|
|
pte = _mali_osk_mem_ioread32(pagedir->page_entries_mapped[pde_index],
|
pte_index * sizeof(u32));
|
|
MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table present: %08x\n"
|
"\t\tPTE: %08x, page %08x is %s\n",
|
fault_addr, pte_addr, pte,
|
MALI_MMU_ENTRY_ADDRESS(pte),
|
pte & MALI_MMU_FLAGS_DEFAULT ? "rw" : "not present"));
|
} else {
|
MALI_DEBUG_PRINT(2, ("\tMMU: %08x: Page table not present: %08x\n",
|
fault_addr, pde));
|
}
|
#else
|
MALI_IGNORE(pagedir);
|
MALI_IGNORE(fault_addr);
|
#endif
|
}
|
|
/* For instrumented */
|
struct dump_info {
|
u32 buffer_left;
|
u32 register_writes_size;
|
u32 page_table_dump_size;
|
u32 *buffer;
|
};
|
|
static _mali_osk_errcode_t writereg(u32 where, u32 what, const char *comment, struct dump_info *info)
|
{
|
if (NULL != info) {
|
info->register_writes_size += sizeof(u32) * 2; /* two 32-bit words */
|
|
if (NULL != info->buffer) {
|
/* check that we have enough space */
|
if (info->buffer_left < sizeof(u32) * 2) MALI_ERROR(_MALI_OSK_ERR_NOMEM);
|
|
*info->buffer = where;
|
info->buffer++;
|
|
*info->buffer = what;
|
info->buffer++;
|
|
info->buffer_left -= sizeof(u32) * 2;
|
}
|
}
|
|
MALI_SUCCESS;
|
}
|
|
static _mali_osk_errcode_t mali_mmu_dump_page(mali_io_address page, u32 phys_addr, struct dump_info *info)
|
{
|
if (NULL != info) {
|
/* 4096 for the page and 4 bytes for the address */
|
const u32 page_size_in_elements = MALI_MMU_PAGE_SIZE / 4;
|
const u32 page_size_in_bytes = MALI_MMU_PAGE_SIZE;
|
const u32 dump_size_in_bytes = MALI_MMU_PAGE_SIZE + 4;
|
|
info->page_table_dump_size += dump_size_in_bytes;
|
|
if (NULL != info->buffer) {
|
if (info->buffer_left < dump_size_in_bytes) MALI_ERROR(_MALI_OSK_ERR_NOMEM);
|
|
*info->buffer = phys_addr;
|
info->buffer++;
|
|
_mali_osk_memcpy(info->buffer, page, page_size_in_bytes);
|
info->buffer += page_size_in_elements;
|
|
info->buffer_left -= dump_size_in_bytes;
|
}
|
}
|
|
MALI_SUCCESS;
|
}
|
|
static _mali_osk_errcode_t dump_mmu_page_table(struct mali_page_directory *pagedir, struct dump_info *info)
|
{
|
MALI_DEBUG_ASSERT_POINTER(pagedir);
|
MALI_DEBUG_ASSERT_POINTER(info);
|
|
if (NULL != pagedir->page_directory_mapped) {
|
int i;
|
|
MALI_CHECK_NO_ERROR(
|
mali_mmu_dump_page(pagedir->page_directory_mapped, pagedir->page_directory, info)
|
);
|
|
for (i = 0; i < 1024; i++) {
|
if (NULL != pagedir->page_entries_mapped[i]) {
|
MALI_CHECK_NO_ERROR(
|
mali_mmu_dump_page(pagedir->page_entries_mapped[i],
|
_mali_osk_mem_ioread32(pagedir->page_directory_mapped,
|
i * sizeof(u32)) & ~MALI_MMU_FLAGS_MASK, info)
|
);
|
}
|
}
|
}
|
|
MALI_SUCCESS;
|
}
|
|
static _mali_osk_errcode_t dump_mmu_registers(struct mali_page_directory *pagedir, struct dump_info *info)
|
{
|
MALI_CHECK_NO_ERROR(writereg(0x00000000, pagedir->page_directory,
|
"set the page directory address", info));
|
MALI_CHECK_NO_ERROR(writereg(0x00000008, 4, "zap???", info));
|
MALI_CHECK_NO_ERROR(writereg(0x00000008, 0, "enable paging", info));
|
MALI_SUCCESS;
|
}
|
|
_mali_osk_errcode_t _mali_ukk_query_mmu_page_table_dump_size(_mali_uk_query_mmu_page_table_dump_size_s *args)
|
{
|
struct dump_info info = { 0, 0, 0, NULL };
|
struct mali_session_data *session_data;
|
|
session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
|
MALI_DEBUG_ASSERT_POINTER(session_data);
|
MALI_DEBUG_ASSERT_POINTER(args);
|
|
MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
|
MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
|
args->size = info.register_writes_size + info.page_table_dump_size;
|
MALI_SUCCESS;
|
}
|
|
_mali_osk_errcode_t _mali_ukk_dump_mmu_page_table(_mali_uk_dump_mmu_page_table_s *args)
|
{
|
struct dump_info info = { 0, 0, 0, NULL };
|
struct mali_session_data *session_data;
|
|
MALI_DEBUG_ASSERT_POINTER(args);
|
|
session_data = (struct mali_session_data *)(uintptr_t)(args->ctx);
|
MALI_DEBUG_ASSERT_POINTER(session_data);
|
|
info.buffer_left = args->size;
|
info.buffer = (u32 *)(uintptr_t)args->buffer;
|
|
args->register_writes = (uintptr_t)info.buffer;
|
MALI_CHECK_NO_ERROR(dump_mmu_registers(session_data->page_directory, &info));
|
|
args->page_table_dump = (uintptr_t)info.buffer;
|
MALI_CHECK_NO_ERROR(dump_mmu_page_table(session_data->page_directory, &info));
|
|
args->register_writes_size = info.register_writes_size;
|
args->page_table_dump_size = info.page_table_dump_size;
|
|
MALI_SUCCESS;
|
}
|
