| .. | .. |
|---|
| 6 | 6 | #include <linux/slab.h> |
|---|
| 7 | 7 | |
|---|
| 8 | 8 | /* |
|---|
| 9 | | - * Functions that deal with pagetables that could be at any level of |
|---|
| 10 | | - * the table need to be passed an "index_size" so they know how to |
|---|
| 11 | | - * handle allocation. For PTE pages (which are linked to a struct |
|---|
| 12 | | - * page for now, and drawn from the main get_free_pages() pool), the |
|---|
| 13 | | - * allocation size will be (2^index_size * sizeof(pointer)) and |
|---|
| 14 | | - * allocations are drawn from the kmem_cache in PGT_CACHE(index_size). |
|---|
| 15 | | - * |
|---|
| 16 | | - * The maximum index size needs to be big enough to allow any |
|---|
| 17 | | - * pagetable sizes we need, but small enough to fit in the low bits of |
|---|
| 18 | | - * any page table pointer. In other words all pagetables, even tiny |
|---|
| 19 | | - * ones, must be aligned to allow at least enough low 0 bits to |
|---|
| 20 | | - * contain this value. This value is also used as a mask, so it must |
|---|
| 21 | | - * be one less than a power of two. |
|---|
| 22 | | - */ |
|---|
| 23 | | -#define MAX_PGTABLE_INDEX_SIZE 0xf |
|---|
| 24 | | - |
|---|
| 25 | | -extern void __bad_pte(pmd_t *pmd); |
|---|
| 26 | | - |
|---|
| 27 | | -extern struct kmem_cache *pgtable_cache[]; |
|---|
| 28 | | -#define PGT_CACHE(shift) ({ \ |
|---|
| 29 | | - BUG_ON(!(shift)); \ |
|---|
| 30 | | - pgtable_cache[(shift) - 1]; \ |
|---|
| 31 | | - }) |
|---|
| 32 | | - |
|---|
| 33 | | -static inline pgd_t *pgd_alloc(struct mm_struct *mm) |
|---|
| 34 | | -{ |
|---|
| 35 | | - return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), |
|---|
| 36 | | - pgtable_gfp_flags(mm, GFP_KERNEL)); |
|---|
| 37 | | -} |
|---|
| 38 | | - |
|---|
| 39 | | -static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) |
|---|
| 40 | | -{ |
|---|
| 41 | | - kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd); |
|---|
| 42 | | -} |
|---|
| 43 | | - |
|---|
| 44 | | -/* |
|---|
| 45 | 9 | * We don't have any real pmd's, and this code never triggers because |
|---|
| 46 | 10 | * the pgd will always be present.. |
|---|
| 47 | 11 | */ |
|---|
| .. | .. |
|---|
| 50 | 14 | #define __pmd_free_tlb(tlb,x,a) do { } while (0) |
|---|
| 51 | 15 | /* #define pgd_populate(mm, pmd, pte) BUG() */ |
|---|
| 52 | 16 | |
|---|
| 53 | | -#ifndef CONFIG_BOOKE |
|---|
| 54 | | - |
|---|
| 55 | 17 | static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, |
|---|
| 56 | 18 | pte_t *pte) |
|---|
| 57 | 19 | { |
|---|
| 58 | | - *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); |
|---|
| 20 | + if (IS_ENABLED(CONFIG_BOOKE)) |
|---|
| 21 | + *pmdp = __pmd((unsigned long)pte | _PMD_PRESENT); |
|---|
| 22 | + else |
|---|
| 23 | + *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); |
|---|
| 59 | 24 | } |
|---|
| 60 | 25 | |
|---|
| 61 | 26 | static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, |
|---|
| 62 | 27 | pgtable_t pte_page) |
|---|
| 63 | 28 | { |
|---|
| 64 | | - *pmdp = __pmd((page_to_pfn(pte_page) << PAGE_SHIFT) | _PMD_USER | |
|---|
| 65 | | - _PMD_PRESENT); |
|---|
| 29 | + if (IS_ENABLED(CONFIG_BOOKE)) |
|---|
| 30 | + *pmdp = __pmd((unsigned long)pte_page | _PMD_PRESENT); |
|---|
| 31 | + else |
|---|
| 32 | + *pmdp = __pmd(__pa(pte_page) | _PMD_USER | _PMD_PRESENT); |
|---|
| 66 | 33 | } |
|---|
| 67 | 34 | |
|---|
| 68 | | -#define pmd_pgtable(pmd) pmd_page(pmd) |
|---|
| 69 | | -#else |
|---|
| 70 | | - |
|---|
| 71 | | -static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, |
|---|
| 72 | | - pte_t *pte) |
|---|
| 73 | | -{ |
|---|
| 74 | | - *pmdp = __pmd((unsigned long)pte | _PMD_PRESENT); |
|---|
| 75 | | -} |
|---|
| 76 | | - |
|---|
| 77 | | -static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, |
|---|
| 78 | | - pgtable_t pte_page) |
|---|
| 79 | | -{ |
|---|
| 80 | | - *pmdp = __pmd((unsigned long)lowmem_page_address(pte_page) | _PMD_PRESENT); |
|---|
| 81 | | -} |
|---|
| 82 | | - |
|---|
| 83 | | -#define pmd_pgtable(pmd) pmd_page(pmd) |
|---|
| 84 | | -#endif |
|---|
| 85 | | - |
|---|
| 86 | | -extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr); |
|---|
| 87 | | -extern pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr); |
|---|
| 88 | | - |
|---|
| 89 | | -static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) |
|---|
| 90 | | -{ |
|---|
| 91 | | - free_page((unsigned long)pte); |
|---|
| 92 | | -} |
|---|
| 93 | | - |
|---|
| 94 | | -static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) |
|---|
| 95 | | -{ |
|---|
| 96 | | - pgtable_page_dtor(ptepage); |
|---|
| 97 | | - __free_page(ptepage); |
|---|
| 98 | | -} |
|---|
| 99 | | - |
|---|
| 100 | | -static inline void pgtable_free(void *table, unsigned index_size) |
|---|
| 101 | | -{ |
|---|
| 102 | | - if (!index_size) { |
|---|
| 103 | | - pgtable_page_dtor(virt_to_page(table)); |
|---|
| 104 | | - free_page((unsigned long)table); |
|---|
| 105 | | - } else { |
|---|
| 106 | | - BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE); |
|---|
| 107 | | - kmem_cache_free(PGT_CACHE(index_size), table); |
|---|
| 108 | | - } |
|---|
| 109 | | -} |
|---|
| 110 | | - |
|---|
| 111 | | -#define check_pgt_cache() do { } while (0) |
|---|
| 112 | | -#define get_hugepd_cache_index(x) (x) |
|---|
| 113 | | - |
|---|
| 114 | | -#ifdef CONFIG_SMP |
|---|
| 115 | | -static inline void pgtable_free_tlb(struct mmu_gather *tlb, |
|---|
| 116 | | - void *table, int shift) |
|---|
| 117 | | -{ |
|---|
| 118 | | - unsigned long pgf = (unsigned long)table; |
|---|
| 119 | | - BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); |
|---|
| 120 | | - pgf |= shift; |
|---|
| 121 | | - tlb_remove_table(tlb, (void *)pgf); |
|---|
| 122 | | -} |
|---|
| 123 | | - |
|---|
| 124 | | -static inline void __tlb_remove_table(void *_table) |
|---|
| 125 | | -{ |
|---|
| 126 | | - void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE); |
|---|
| 127 | | - unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE; |
|---|
| 128 | | - |
|---|
| 129 | | - pgtable_free(table, shift); |
|---|
| 130 | | -} |
|---|
| 131 | | -#else |
|---|
| 132 | | -static inline void pgtable_free_tlb(struct mmu_gather *tlb, |
|---|
| 133 | | - void *table, int shift) |
|---|
| 134 | | -{ |
|---|
| 135 | | - pgtable_free(table, shift); |
|---|
| 136 | | -} |
|---|
| 137 | | -#endif |
|---|
| 138 | | - |
|---|
| 139 | | -static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, |
|---|
| 140 | | - unsigned long address) |
|---|
| 141 | | -{ |
|---|
| 142 | | - tlb_flush_pgtable(tlb, address); |
|---|
| 143 | | - pgtable_free_tlb(tlb, page_address(table), 0); |
|---|
| 144 | | -} |
|---|
| 145 | 35 | #endif /* _ASM_POWERPC_PGALLOC_32_H */ |
|---|