// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms * of such GNU license. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. * */ #include #include #include #include #include #include /** * lock_region() - Generate lockaddr to lock memory region in MMU * @pfn: Starting page frame number of the region to lock * @num_pages: Number of pages to lock. It must be greater than 0. * @lockaddr: Address and size of memory region to lock * * The lockaddr value is a combination of the starting address and * the size of the region that encompasses all the memory pages to lock. * * The size is expressed as a logarithm: it is represented in a way * that is compatible with the HW specification and it also determines * how many of the lowest bits of the address are cleared. * * Return: 0 if success, or an error code on failure. */ static int lock_region(u64 pfn, u32 num_pages, u64 *lockaddr) { const u64 lockaddr_base = pfn << PAGE_SHIFT; u64 lockaddr_size_log2, region_frame_number_start, region_frame_number_end; if (num_pages == 0) return -EINVAL; /* The size is expressed as a logarithm and should take into account * the possibility that some pages might spill into the next region. */ lockaddr_size_log2 = fls(num_pages) + PAGE_SHIFT - 1; /* Round up if the number of pages is not a power of 2. */ if (num_pages != ((u32)1 << (lockaddr_size_log2 - PAGE_SHIFT))) lockaddr_size_log2 += 1; /* Round up if some memory pages spill into the next region. */ region_frame_number_start = pfn >> (lockaddr_size_log2 - PAGE_SHIFT); region_frame_number_end = (pfn + num_pages - 1) >> (lockaddr_size_log2 - PAGE_SHIFT); if (region_frame_number_start < region_frame_number_end) lockaddr_size_log2 += 1; /* Represent the size according to the HW specification. */ lockaddr_size_log2 = MAX(lockaddr_size_log2, KBASE_LOCK_REGION_MIN_SIZE_LOG2); if (lockaddr_size_log2 > KBASE_LOCK_REGION_MAX_SIZE_LOG2) return -EINVAL; /* The lowest bits are cleared and then set to size - 1 to represent * the size in a way that is compatible with the HW specification. */ *lockaddr = lockaddr_base & ~((1ull << lockaddr_size_log2) - 1); *lockaddr |= lockaddr_size_log2 - 1; return 0; } static int wait_ready(struct kbase_device *kbdev, unsigned int as_nr) { unsigned int max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; u32 val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS)); /* Wait for the MMU status to indicate there is no active command, in * case one is pending. Do not log remaining register accesses. */ while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) val = kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS)); if (max_loops == 0) { dev_err(kbdev->dev, "AS_ACTIVE bit stuck, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n"); return -1; } /* If waiting in loop was performed, log last read value. */ if (KBASE_AS_INACTIVE_MAX_LOOPS - 1 > max_loops) kbase_reg_read(kbdev, MMU_AS_REG(as_nr, AS_STATUS)); return 0; } static int write_cmd(struct kbase_device *kbdev, int as_nr, u32 cmd) { int status; /* write AS_COMMAND when MMU is ready to accept another command */ status = wait_ready(kbdev, as_nr); if (status == 0) kbase_reg_write(kbdev, MMU_AS_REG(as_nr, AS_COMMAND), cmd); return status; } void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) { struct kbase_mmu_setup *current_setup = &as->current_setup; u64 transcfg = 0; transcfg = current_setup->transcfg; /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK * Clear PTW_MEMATTR bits */ transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; /* Enable correct PTW_MEMATTR bits */ transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; /* Ensure page-tables reads use read-allocate cache-policy in * the L2 */ transcfg |= AS_TRANSCFG_R_ALLOCATE; if (kbdev->system_coherency != COHERENCY_NONE) { /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) * Clear PTW_SH bits */ transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); /* Enable correct PTW_SH bits */ transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); } kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), transcfg); kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), (transcfg >> 32) & 0xFFFFFFFFUL); kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO), current_setup->transtab & 0xFFFFFFFFUL); kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI), (current_setup->transtab >> 32) & 0xFFFFFFFFUL); kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_LO), current_setup->memattr & 0xFFFFFFFFUL); kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_MEMATTR_HI), (current_setup->memattr >> 32) & 0xFFFFFFFFUL); KBASE_TLSTREAM_TL_ATTRIB_AS_CONFIG(kbdev, as, current_setup->transtab, current_setup->memattr, transcfg); write_cmd(kbdev, as->number, AS_COMMAND_UPDATE); } int kbase_mmu_hw_do_operation(struct kbase_device *kbdev, struct kbase_as *as, u64 vpfn, u32 nr, u32 op, unsigned int handling_irq) { int ret; lockdep_assert_held(&kbdev->mmu_hw_mutex); if (op == AS_COMMAND_UNLOCK) { /* Unlock doesn't require a lock first */ ret = write_cmd(kbdev, as->number, AS_COMMAND_UNLOCK); } else { u64 lock_addr; ret = lock_region(vpfn, nr, &lock_addr); if (!ret) { /* Lock the region that needs to be updated */ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_LO), lock_addr & 0xFFFFFFFFUL); kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_LOCKADDR_HI), (lock_addr >> 32) & 0xFFFFFFFFUL); write_cmd(kbdev, as->number, AS_COMMAND_LOCK); /* Run the MMU operation */ write_cmd(kbdev, as->number, op); /* Wait for the flush to complete */ ret = wait_ready(kbdev, as->number); } } return ret; } void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, enum kbase_mmu_fault_type type) { unsigned long flags; u32 pf_bf_mask; spin_lock_irqsave(&kbdev->mmu_mask_change, flags); /* * A reset is in-flight and we're flushing the IRQ + bottom half * so don't update anything as it could race with the reset code. */ if (kbdev->irq_reset_flush) goto unlock; /* Clear the page (and bus fault IRQ as well in case one occurred) */ pf_bf_mask = MMU_PAGE_FAULT(as->number); #if !MALI_USE_CSF if (type == KBASE_MMU_FAULT_TYPE_BUS || type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) pf_bf_mask |= MMU_BUS_ERROR(as->number); #endif kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask); unlock: spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); } void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, enum kbase_mmu_fault_type type) { unsigned long flags; u32 irq_mask; /* Enable the page fault IRQ * (and bus fault IRQ as well in case one occurred) */ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); /* * A reset is in-flight and we're flushing the IRQ + bottom half * so don't update anything as it could race with the reset code. */ if (kbdev->irq_reset_flush) goto unlock; irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)) | MMU_PAGE_FAULT(as->number); #if !MALI_USE_CSF if (type == KBASE_MMU_FAULT_TYPE_BUS || type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) irq_mask |= MMU_BUS_ERROR(as->number); #endif kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask); unlock: spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); }