| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * Set up the VMAs to tell the VM about the vDSO. |
|---|
| 3 | 4 | * Copyright 2007 Andi Kleen, SUSE Labs. |
|---|
| 4 | | - * Subject to the GPL, v.2 |
|---|
| 5 | 5 | */ |
|---|
| 6 | 6 | |
|---|
| 7 | 7 | /* |
|---|
| .. | .. |
|---|
| 16 | 16 | #include <linux/linkage.h> |
|---|
| 17 | 17 | #include <linux/random.h> |
|---|
| 18 | 18 | #include <linux/elf.h> |
|---|
| 19 | +#include <asm/cacheflush.h> |
|---|
| 20 | +#include <asm/spitfire.h> |
|---|
| 19 | 21 | #include <asm/vdso.h> |
|---|
| 20 | 22 | #include <asm/vvar.h> |
|---|
| 21 | 23 | #include <asm/page.h> |
|---|
| .. | .. |
|---|
| 40 | 42 | |
|---|
| 41 | 43 | struct vvar_data *vvar_data; |
|---|
| 42 | 44 | |
|---|
| 43 | | -#define SAVE_INSTR_SIZE 4 |
|---|
| 45 | +struct vdso_elfinfo32 { |
|---|
| 46 | + Elf32_Ehdr *hdr; |
|---|
| 47 | + Elf32_Sym *dynsym; |
|---|
| 48 | + unsigned long dynsymsize; |
|---|
| 49 | + const char *dynstr; |
|---|
| 50 | + unsigned long text; |
|---|
| 51 | +}; |
|---|
| 52 | + |
|---|
| 53 | +struct vdso_elfinfo64 { |
|---|
| 54 | + Elf64_Ehdr *hdr; |
|---|
| 55 | + Elf64_Sym *dynsym; |
|---|
| 56 | + unsigned long dynsymsize; |
|---|
| 57 | + const char *dynstr; |
|---|
| 58 | + unsigned long text; |
|---|
| 59 | +}; |
|---|
| 60 | + |
|---|
| 61 | +struct vdso_elfinfo { |
|---|
| 62 | + union { |
|---|
| 63 | + struct vdso_elfinfo32 elf32; |
|---|
| 64 | + struct vdso_elfinfo64 elf64; |
|---|
| 65 | + } u; |
|---|
| 66 | +}; |
|---|
| 67 | + |
|---|
| 68 | +static void *one_section64(struct vdso_elfinfo64 *e, const char *name, |
|---|
| 69 | + unsigned long *size) |
|---|
| 70 | +{ |
|---|
| 71 | + const char *snames; |
|---|
| 72 | + Elf64_Shdr *shdrs; |
|---|
| 73 | + unsigned int i; |
|---|
| 74 | + |
|---|
| 75 | + shdrs = (void *)e->hdr + e->hdr->e_shoff; |
|---|
| 76 | + snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset; |
|---|
| 77 | + for (i = 1; i < e->hdr->e_shnum; i++) { |
|---|
| 78 | + if (!strcmp(snames+shdrs[i].sh_name, name)) { |
|---|
| 79 | + if (size) |
|---|
| 80 | + *size = shdrs[i].sh_size; |
|---|
| 81 | + return (void *)e->hdr + shdrs[i].sh_offset; |
|---|
| 82 | + } |
|---|
| 83 | + } |
|---|
| 84 | + return NULL; |
|---|
| 85 | +} |
|---|
| 86 | + |
|---|
| 87 | +static int find_sections64(const struct vdso_image *image, struct vdso_elfinfo *_e) |
|---|
| 88 | +{ |
|---|
| 89 | + struct vdso_elfinfo64 *e = &_e->u.elf64; |
|---|
| 90 | + |
|---|
| 91 | + e->hdr = image->data; |
|---|
| 92 | + e->dynsym = one_section64(e, ".dynsym", &e->dynsymsize); |
|---|
| 93 | + e->dynstr = one_section64(e, ".dynstr", NULL); |
|---|
| 94 | + |
|---|
| 95 | + if (!e->dynsym || !e->dynstr) { |
|---|
| 96 | + pr_err("VDSO64: Missing symbol sections.\n"); |
|---|
| 97 | + return -ENODEV; |
|---|
| 98 | + } |
|---|
| 99 | + return 0; |
|---|
| 100 | +} |
|---|
| 101 | + |
|---|
| 102 | +static Elf64_Sym *find_sym64(const struct vdso_elfinfo64 *e, const char *name) |
|---|
| 103 | +{ |
|---|
| 104 | + unsigned int i; |
|---|
| 105 | + |
|---|
| 106 | + for (i = 0; i < (e->dynsymsize / sizeof(Elf64_Sym)); i++) { |
|---|
| 107 | + Elf64_Sym *s = &e->dynsym[i]; |
|---|
| 108 | + if (s->st_name == 0) |
|---|
| 109 | + continue; |
|---|
| 110 | + if (!strcmp(e->dynstr + s->st_name, name)) |
|---|
| 111 | + return s; |
|---|
| 112 | + } |
|---|
| 113 | + return NULL; |
|---|
| 114 | +} |
|---|
| 115 | + |
|---|
| 116 | +static int patchsym64(struct vdso_elfinfo *_e, const char *orig, |
|---|
| 117 | + const char *new) |
|---|
| 118 | +{ |
|---|
| 119 | + struct vdso_elfinfo64 *e = &_e->u.elf64; |
|---|
| 120 | + Elf64_Sym *osym = find_sym64(e, orig); |
|---|
| 121 | + Elf64_Sym *nsym = find_sym64(e, new); |
|---|
| 122 | + |
|---|
| 123 | + if (!nsym || !osym) { |
|---|
| 124 | + pr_err("VDSO64: Missing symbols.\n"); |
|---|
| 125 | + return -ENODEV; |
|---|
| 126 | + } |
|---|
| 127 | + osym->st_value = nsym->st_value; |
|---|
| 128 | + osym->st_size = nsym->st_size; |
|---|
| 129 | + osym->st_info = nsym->st_info; |
|---|
| 130 | + osym->st_other = nsym->st_other; |
|---|
| 131 | + osym->st_shndx = nsym->st_shndx; |
|---|
| 132 | + |
|---|
| 133 | + return 0; |
|---|
| 134 | +} |
|---|
| 135 | + |
|---|
| 136 | +static void *one_section32(struct vdso_elfinfo32 *e, const char *name, |
|---|
| 137 | + unsigned long *size) |
|---|
| 138 | +{ |
|---|
| 139 | + const char *snames; |
|---|
| 140 | + Elf32_Shdr *shdrs; |
|---|
| 141 | + unsigned int i; |
|---|
| 142 | + |
|---|
| 143 | + shdrs = (void *)e->hdr + e->hdr->e_shoff; |
|---|
| 144 | + snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset; |
|---|
| 145 | + for (i = 1; i < e->hdr->e_shnum; i++) { |
|---|
| 146 | + if (!strcmp(snames+shdrs[i].sh_name, name)) { |
|---|
| 147 | + if (size) |
|---|
| 148 | + *size = shdrs[i].sh_size; |
|---|
| 149 | + return (void *)e->hdr + shdrs[i].sh_offset; |
|---|
| 150 | + } |
|---|
| 151 | + } |
|---|
| 152 | + return NULL; |
|---|
| 153 | +} |
|---|
| 154 | + |
|---|
| 155 | +static int find_sections32(const struct vdso_image *image, struct vdso_elfinfo *_e) |
|---|
| 156 | +{ |
|---|
| 157 | + struct vdso_elfinfo32 *e = &_e->u.elf32; |
|---|
| 158 | + |
|---|
| 159 | + e->hdr = image->data; |
|---|
| 160 | + e->dynsym = one_section32(e, ".dynsym", &e->dynsymsize); |
|---|
| 161 | + e->dynstr = one_section32(e, ".dynstr", NULL); |
|---|
| 162 | + |
|---|
| 163 | + if (!e->dynsym || !e->dynstr) { |
|---|
| 164 | + pr_err("VDSO32: Missing symbol sections.\n"); |
|---|
| 165 | + return -ENODEV; |
|---|
| 166 | + } |
|---|
| 167 | + return 0; |
|---|
| 168 | +} |
|---|
| 169 | + |
|---|
| 170 | +static Elf32_Sym *find_sym32(const struct vdso_elfinfo32 *e, const char *name) |
|---|
| 171 | +{ |
|---|
| 172 | + unsigned int i; |
|---|
| 173 | + |
|---|
| 174 | + for (i = 0; i < (e->dynsymsize / sizeof(Elf32_Sym)); i++) { |
|---|
| 175 | + Elf32_Sym *s = &e->dynsym[i]; |
|---|
| 176 | + if (s->st_name == 0) |
|---|
| 177 | + continue; |
|---|
| 178 | + if (!strcmp(e->dynstr + s->st_name, name)) |
|---|
| 179 | + return s; |
|---|
| 180 | + } |
|---|
| 181 | + return NULL; |
|---|
| 182 | +} |
|---|
| 183 | + |
|---|
| 184 | +static int patchsym32(struct vdso_elfinfo *_e, const char *orig, |
|---|
| 185 | + const char *new) |
|---|
| 186 | +{ |
|---|
| 187 | + struct vdso_elfinfo32 *e = &_e->u.elf32; |
|---|
| 188 | + Elf32_Sym *osym = find_sym32(e, orig); |
|---|
| 189 | + Elf32_Sym *nsym = find_sym32(e, new); |
|---|
| 190 | + |
|---|
| 191 | + if (!nsym || !osym) { |
|---|
| 192 | + pr_err("VDSO32: Missing symbols.\n"); |
|---|
| 193 | + return -ENODEV; |
|---|
| 194 | + } |
|---|
| 195 | + osym->st_value = nsym->st_value; |
|---|
| 196 | + osym->st_size = nsym->st_size; |
|---|
| 197 | + osym->st_info = nsym->st_info; |
|---|
| 198 | + osym->st_other = nsym->st_other; |
|---|
| 199 | + osym->st_shndx = nsym->st_shndx; |
|---|
| 200 | + |
|---|
| 201 | + return 0; |
|---|
| 202 | +} |
|---|
| 203 | + |
|---|
| 204 | +static int find_sections(const struct vdso_image *image, struct vdso_elfinfo *e, |
|---|
| 205 | + bool elf64) |
|---|
| 206 | +{ |
|---|
| 207 | + if (elf64) |
|---|
| 208 | + return find_sections64(image, e); |
|---|
| 209 | + else |
|---|
| 210 | + return find_sections32(image, e); |
|---|
| 211 | +} |
|---|
| 212 | + |
|---|
| 213 | +static int patch_one_symbol(struct vdso_elfinfo *e, const char *orig, |
|---|
| 214 | + const char *new_target, bool elf64) |
|---|
| 215 | +{ |
|---|
| 216 | + if (elf64) |
|---|
| 217 | + return patchsym64(e, orig, new_target); |
|---|
| 218 | + else |
|---|
| 219 | + return patchsym32(e, orig, new_target); |
|---|
| 220 | +} |
|---|
| 221 | + |
|---|
| 222 | +static int stick_patch(const struct vdso_image *image, struct vdso_elfinfo *e, bool elf64) |
|---|
| 223 | +{ |
|---|
| 224 | + int err; |
|---|
| 225 | + |
|---|
| 226 | + err = find_sections(image, e, elf64); |
|---|
| 227 | + if (err) |
|---|
| 228 | + return err; |
|---|
| 229 | + |
|---|
| 230 | + err = patch_one_symbol(e, |
|---|
| 231 | + "__vdso_gettimeofday", |
|---|
| 232 | + "__vdso_gettimeofday_stick", elf64); |
|---|
| 233 | + if (err) |
|---|
| 234 | + return err; |
|---|
| 235 | + |
|---|
| 236 | + return patch_one_symbol(e, |
|---|
| 237 | + "__vdso_clock_gettime", |
|---|
| 238 | + "__vdso_clock_gettime_stick", elf64); |
|---|
| 239 | + return 0; |
|---|
| 240 | +} |
|---|
| 44 | 241 | |
|---|
| 45 | 242 | /* |
|---|
| 46 | 243 | * Allocate pages for the vdso and vvar, and copy in the vdso text from the |
|---|
| 47 | 244 | * kernel image. |
|---|
| 48 | 245 | */ |
|---|
| 49 | 246 | int __init init_vdso_image(const struct vdso_image *image, |
|---|
| 50 | | - struct vm_special_mapping *vdso_mapping) |
|---|
| 247 | + struct vm_special_mapping *vdso_mapping, bool elf64) |
|---|
| 51 | 248 | { |
|---|
| 52 | | - int i; |
|---|
| 53 | | - struct page *dp, **dpp = NULL; |
|---|
| 54 | | - int dnpages = 0; |
|---|
| 55 | | - struct page *cp, **cpp = NULL; |
|---|
| 56 | 249 | int cnpages = (image->size) / PAGE_SIZE; |
|---|
| 250 | + struct page *dp, **dpp = NULL; |
|---|
| 251 | + struct page *cp, **cpp = NULL; |
|---|
| 252 | + struct vdso_elfinfo ei; |
|---|
| 253 | + int i, dnpages = 0; |
|---|
| 254 | + |
|---|
| 255 | + if (tlb_type != spitfire) { |
|---|
| 256 | + int err = stick_patch(image, &ei, elf64); |
|---|
| 257 | + if (err) |
|---|
| 258 | + return err; |
|---|
| 259 | + } |
|---|
| 57 | 260 | |
|---|
| 58 | 261 | /* |
|---|
| 59 | 262 | * First, the vdso text. This is initialied data, an integral number of |
|---|
| .. | .. |
|---|
| 67 | 270 | |
|---|
| 68 | 271 | if (!cpp) |
|---|
| 69 | 272 | goto oom; |
|---|
| 70 | | - |
|---|
| 71 | | - if (vdso_fix_stick) { |
|---|
| 72 | | - /* |
|---|
| 73 | | - * If the system uses %tick instead of %stick, patch the VDSO |
|---|
| 74 | | - * with instruction reading %tick instead of %stick. |
|---|
| 75 | | - */ |
|---|
| 76 | | - unsigned int j, k = SAVE_INSTR_SIZE; |
|---|
| 77 | | - unsigned char *data = image->data; |
|---|
| 78 | | - |
|---|
| 79 | | - for (j = image->sym_vread_tick_patch_start; |
|---|
| 80 | | - j < image->sym_vread_tick_patch_end; j++) { |
|---|
| 81 | | - |
|---|
| 82 | | - data[image->sym_vread_tick + k] = data[j]; |
|---|
| 83 | | - k++; |
|---|
| 84 | | - } |
|---|
| 85 | | - } |
|---|
| 86 | 273 | |
|---|
| 87 | 274 | for (i = 0; i < cnpages; i++) { |
|---|
| 88 | 275 | cp = alloc_page(GFP_KERNEL); |
|---|
| .. | .. |
|---|
| 146 | 333 | { |
|---|
| 147 | 334 | int err = 0; |
|---|
| 148 | 335 | #ifdef CONFIG_SPARC64 |
|---|
| 149 | | - err = init_vdso_image(&vdso_image_64_builtin, &vdso_mapping64); |
|---|
| 336 | + err = init_vdso_image(&vdso_image_64_builtin, &vdso_mapping64, true); |
|---|
| 150 | 337 | if (err) |
|---|
| 151 | 338 | return err; |
|---|
| 152 | 339 | #endif |
|---|
| 153 | 340 | |
|---|
| 154 | 341 | #ifdef CONFIG_COMPAT |
|---|
| 155 | | - err = init_vdso_image(&vdso_image_32_builtin, &vdso_mapping32); |
|---|
| 342 | + err = init_vdso_image(&vdso_image_32_builtin, &vdso_mapping32, false); |
|---|
| 156 | 343 | #endif |
|---|
| 157 | 344 | return err; |
|---|
| 158 | 345 | |
|---|
| .. | .. |
|---|
| 179 | 366 | unsigned long text_start, addr = 0; |
|---|
| 180 | 367 | int ret = 0; |
|---|
| 181 | 368 | |
|---|
| 182 | | - down_write(&mm->mmap_sem); |
|---|
| 369 | + mmap_write_lock(mm); |
|---|
| 183 | 370 | |
|---|
| 184 | 371 | /* |
|---|
| 185 | 372 | * First, get an unmapped region: then randomize it, and make sure that |
|---|
| .. | .. |
|---|
| 235 | 422 | if (ret) |
|---|
| 236 | 423 | current->mm->context.vdso = NULL; |
|---|
| 237 | 424 | |
|---|
| 238 | | - up_write(&mm->mmap_sem); |
|---|
| 425 | + mmap_write_unlock(mm); |
|---|
| 239 | 426 | return ret; |
|---|
| 240 | 427 | } |
|---|
| 241 | 428 | |
|---|