| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * Adapted from arm64 version. |
|---|
| 3 | 4 | * |
|---|
| 4 | 5 | * Copyright (C) 2012 ARM Limited |
|---|
| 5 | 6 | * Copyright (C) 2015 Mentor Graphics Corporation. |
|---|
| 6 | | - * |
|---|
| 7 | | - * This program is free software; you can redistribute it and/or modify |
|---|
| 8 | | - * it under the terms of the GNU General Public License version 2 as |
|---|
| 9 | | - * published by the Free Software Foundation. |
|---|
| 10 | | - * |
|---|
| 11 | | - * This program is distributed in the hope that it will be useful, |
|---|
| 12 | | - * but WITHOUT ANY WARRANTY; without even the implied warranty of |
|---|
| 13 | | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|---|
| 14 | | - * GNU General Public License for more details. |
|---|
| 15 | | - * |
|---|
| 16 | | - * You should have received a copy of the GNU General Public License |
|---|
| 17 | | - * along with this program. If not, see <http://www.gnu.org/licenses/>. |
|---|
| 18 | 7 | */ |
|---|
| 19 | 8 | |
|---|
| 20 | 9 | #include <linux/cache.h> |
|---|
| .. | .. |
|---|
| 34 | 23 | #include <asm/vdso.h> |
|---|
| 35 | 24 | #include <asm/vdso_datapage.h> |
|---|
| 36 | 25 | #include <clocksource/arm_arch_timer.h> |
|---|
| 26 | +#include <vdso/helpers.h> |
|---|
| 27 | +#include <vdso/vsyscall.h> |
|---|
| 37 | 28 | |
|---|
| 38 | 29 | #define MAX_SYMNAME 64 |
|---|
| 39 | 30 | |
|---|
| .. | .. |
|---|
| 48 | 39 | * The VDSO data page. |
|---|
| 49 | 40 | */ |
|---|
| 50 | 41 | static union vdso_data_store vdso_data_store __page_aligned_data; |
|---|
| 51 | | -static struct vdso_data *vdso_data = &vdso_data_store.data; |
|---|
| 42 | +struct vdso_data *vdso_data = vdso_data_store.data; |
|---|
| 52 | 43 | |
|---|
| 53 | 44 | static struct page *vdso_data_page __ro_after_init; |
|---|
| 54 | 45 | static const struct vm_special_mapping vdso_data_mapping = { |
|---|
| .. | .. |
|---|
| 88 | 79 | /* Cached result of boot-time check for whether the arch timer exists, |
|---|
| 89 | 80 | * and if so, whether the virtual counter is useable. |
|---|
| 90 | 81 | */ |
|---|
| 91 | | -static bool cntvct_ok __ro_after_init; |
|---|
| 82 | +bool cntvct_ok __ro_after_init; |
|---|
| 92 | 83 | |
|---|
| 93 | 84 | static bool __init cntvct_functional(void) |
|---|
| 94 | 85 | { |
|---|
| .. | .. |
|---|
| 193 | 184 | if (!cntvct_ok) { |
|---|
| 194 | 185 | vdso_nullpatch_one(&einfo, "__vdso_gettimeofday"); |
|---|
| 195 | 186 | vdso_nullpatch_one(&einfo, "__vdso_clock_gettime"); |
|---|
| 187 | + vdso_nullpatch_one(&einfo, "__vdso_clock_gettime64"); |
|---|
| 196 | 188 | } |
|---|
| 197 | 189 | } |
|---|
| 198 | 190 | |
|---|
| .. | .. |
|---|
| 249 | 241 | return PTR_ERR_OR_ZERO(vma); |
|---|
| 250 | 242 | } |
|---|
| 251 | 243 | |
|---|
| 252 | | -/* assumes mmap_sem is write-locked */ |
|---|
| 244 | +/* assumes mmap_lock is write-locked */ |
|---|
| 253 | 245 | void arm_install_vdso(struct mm_struct *mm, unsigned long addr) |
|---|
| 254 | 246 | { |
|---|
| 255 | 247 | struct vm_area_struct *vma; |
|---|
| .. | .. |
|---|
| 275 | 267 | mm->context.vdso = addr; |
|---|
| 276 | 268 | } |
|---|
| 277 | 269 | |
|---|
| 278 | | -static void vdso_write_begin(struct vdso_data *vdata) |
|---|
| 279 | | -{ |
|---|
| 280 | | - ++vdso_data->seq_count; |
|---|
| 281 | | - smp_wmb(); /* Pairs with smp_rmb in vdso_read_retry */ |
|---|
| 282 | | -} |
|---|
| 283 | | - |
|---|
| 284 | | -static void vdso_write_end(struct vdso_data *vdata) |
|---|
| 285 | | -{ |
|---|
| 286 | | - smp_wmb(); /* Pairs with smp_rmb in vdso_read_begin */ |
|---|
| 287 | | - ++vdso_data->seq_count; |
|---|
| 288 | | -} |
|---|
| 289 | | - |
|---|
| 290 | | -static bool tk_is_cntvct(const struct timekeeper *tk) |
|---|
| 291 | | -{ |
|---|
| 292 | | - if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER)) |
|---|
| 293 | | - return false; |
|---|
| 294 | | - |
|---|
| 295 | | - if (!tk->tkr_mono.clock->archdata.vdso_direct) |
|---|
| 296 | | - return false; |
|---|
| 297 | | - |
|---|
| 298 | | - return true; |
|---|
| 299 | | -} |
|---|
| 300 | | - |
|---|
| 301 | | -/** |
|---|
| 302 | | - * update_vsyscall - update the vdso data page |
|---|
| 303 | | - * |
|---|
| 304 | | - * Increment the sequence counter, making it odd, indicating to |
|---|
| 305 | | - * userspace that an update is in progress. Update the fields used |
|---|
| 306 | | - * for coarse clocks and, if the architected system timer is in use, |
|---|
| 307 | | - * the fields used for high precision clocks. Increment the sequence |
|---|
| 308 | | - * counter again, making it even, indicating to userspace that the |
|---|
| 309 | | - * update is finished. |
|---|
| 310 | | - * |
|---|
| 311 | | - * Userspace is expected to sample seq_count before reading any other |
|---|
| 312 | | - * fields from the data page. If seq_count is odd, userspace is |
|---|
| 313 | | - * expected to wait until it becomes even. After copying data from |
|---|
| 314 | | - * the page, userspace must sample seq_count again; if it has changed |
|---|
| 315 | | - * from its previous value, userspace must retry the whole sequence. |
|---|
| 316 | | - * |
|---|
| 317 | | - * Calls to update_vsyscall are serialized by the timekeeping core. |
|---|
| 318 | | - */ |
|---|
| 319 | | -void update_vsyscall(struct timekeeper *tk) |
|---|
| 320 | | -{ |
|---|
| 321 | | - struct timespec64 *wtm = &tk->wall_to_monotonic; |
|---|
| 322 | | - |
|---|
| 323 | | - if (!cntvct_ok) { |
|---|
| 324 | | - /* The entry points have been zeroed, so there is no |
|---|
| 325 | | - * point in updating the data page. |
|---|
| 326 | | - */ |
|---|
| 327 | | - return; |
|---|
| 328 | | - } |
|---|
| 329 | | - |
|---|
| 330 | | - vdso_write_begin(vdso_data); |
|---|
| 331 | | - |
|---|
| 332 | | - vdso_data->tk_is_cntvct = tk_is_cntvct(tk); |
|---|
| 333 | | - vdso_data->xtime_coarse_sec = tk->xtime_sec; |
|---|
| 334 | | - vdso_data->xtime_coarse_nsec = (u32)(tk->tkr_mono.xtime_nsec >> |
|---|
| 335 | | - tk->tkr_mono.shift); |
|---|
| 336 | | - vdso_data->wtm_clock_sec = wtm->tv_sec; |
|---|
| 337 | | - vdso_data->wtm_clock_nsec = wtm->tv_nsec; |
|---|
| 338 | | - |
|---|
| 339 | | - if (vdso_data->tk_is_cntvct) { |
|---|
| 340 | | - vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last; |
|---|
| 341 | | - vdso_data->xtime_clock_sec = tk->xtime_sec; |
|---|
| 342 | | - vdso_data->xtime_clock_snsec = tk->tkr_mono.xtime_nsec; |
|---|
| 343 | | - vdso_data->cs_mult = tk->tkr_mono.mult; |
|---|
| 344 | | - vdso_data->cs_shift = tk->tkr_mono.shift; |
|---|
| 345 | | - vdso_data->cs_mask = tk->tkr_mono.mask; |
|---|
| 346 | | - } |
|---|
| 347 | | - |
|---|
| 348 | | - vdso_write_end(vdso_data); |
|---|
| 349 | | - |
|---|
| 350 | | - flush_dcache_page(virt_to_page(vdso_data)); |
|---|
| 351 | | -} |
|---|
| 352 | | - |
|---|
| 353 | | -void update_vsyscall_tz(void) |
|---|
| 354 | | -{ |
|---|
| 355 | | - vdso_data->tz_minuteswest = sys_tz.tz_minuteswest; |
|---|
| 356 | | - vdso_data->tz_dsttime = sys_tz.tz_dsttime; |
|---|
| 357 | | - flush_dcache_page(virt_to_page(vdso_data)); |
|---|
| 358 | | -} |
|---|