| .. | .. |
|---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0-or-later */ |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * Generic barrier definitions. |
|---|
| 3 | 4 | * |
|---|
| .. | .. |
|---|
| 6 | 7 | * |
|---|
| 7 | 8 | * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. |
|---|
| 8 | 9 | * Written by David Howells (dhowells@redhat.com) |
|---|
| 9 | | - * |
|---|
| 10 | | - * This program is free software; you can redistribute it and/or |
|---|
| 11 | | - * modify it under the terms of the GNU General Public Licence |
|---|
| 12 | | - * as published by the Free Software Foundation; either version |
|---|
| 13 | | - * 2 of the Licence, or (at your option) any later version. |
|---|
| 14 | 10 | */ |
|---|
| 15 | 11 | #ifndef __ASM_GENERIC_BARRIER_H |
|---|
| 16 | 12 | #define __ASM_GENERIC_BARRIER_H |
|---|
| .. | .. |
|---|
| 18 | 14 | #ifndef __ASSEMBLY__ |
|---|
| 19 | 15 | |
|---|
| 20 | 16 | #include <linux/compiler.h> |
|---|
| 17 | +#include <asm/rwonce.h> |
|---|
| 21 | 18 | |
|---|
| 22 | 19 | #ifndef nop |
|---|
| 23 | 20 | #define nop() asm volatile ("nop") |
|---|
| .. | .. |
|---|
| 50 | 47 | #define dma_wmb() wmb() |
|---|
| 51 | 48 | #endif |
|---|
| 52 | 49 | |
|---|
| 53 | | -#ifndef read_barrier_depends |
|---|
| 54 | | -#define read_barrier_depends() do { } while (0) |
|---|
| 55 | | -#endif |
|---|
| 56 | | - |
|---|
| 57 | 50 | #ifndef __smp_mb |
|---|
| 58 | 51 | #define __smp_mb() mb() |
|---|
| 59 | 52 | #endif |
|---|
| .. | .. |
|---|
| 64 | 57 | |
|---|
| 65 | 58 | #ifndef __smp_wmb |
|---|
| 66 | 59 | #define __smp_wmb() wmb() |
|---|
| 67 | | -#endif |
|---|
| 68 | | - |
|---|
| 69 | | -#ifndef __smp_read_barrier_depends |
|---|
| 70 | | -#define __smp_read_barrier_depends() read_barrier_depends() |
|---|
| 71 | 60 | #endif |
|---|
| 72 | 61 | |
|---|
| 73 | 62 | #ifdef CONFIG_SMP |
|---|
| .. | .. |
|---|
| 84 | 73 | #define smp_wmb() __smp_wmb() |
|---|
| 85 | 74 | #endif |
|---|
| 86 | 75 | |
|---|
| 87 | | -#ifndef smp_read_barrier_depends |
|---|
| 88 | | -#define smp_read_barrier_depends() __smp_read_barrier_depends() |
|---|
| 89 | | -#endif |
|---|
| 90 | | - |
|---|
| 91 | 76 | #else /* !CONFIG_SMP */ |
|---|
| 92 | 77 | |
|---|
| 93 | 78 | #ifndef smp_mb |
|---|
| .. | .. |
|---|
| 100 | 85 | |
|---|
| 101 | 86 | #ifndef smp_wmb |
|---|
| 102 | 87 | #define smp_wmb() barrier() |
|---|
| 103 | | -#endif |
|---|
| 104 | | - |
|---|
| 105 | | -#ifndef smp_read_barrier_depends |
|---|
| 106 | | -#define smp_read_barrier_depends() do { } while (0) |
|---|
| 107 | 88 | #endif |
|---|
| 108 | 89 | |
|---|
| 109 | 90 | #endif /* CONFIG_SMP */ |
|---|
| .. | .. |
|---|
| 132 | 113 | #ifndef __smp_load_acquire |
|---|
| 133 | 114 | #define __smp_load_acquire(p) \ |
|---|
| 134 | 115 | ({ \ |
|---|
| 135 | | - typeof(*p) ___p1 = READ_ONCE(*p); \ |
|---|
| 116 | + __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ |
|---|
| 136 | 117 | compiletime_assert_atomic_type(*p); \ |
|---|
| 137 | 118 | __smp_mb(); \ |
|---|
| 138 | | - ___p1; \ |
|---|
| 119 | + (typeof(*p))___p1; \ |
|---|
| 139 | 120 | }) |
|---|
| 140 | 121 | #endif |
|---|
| 141 | 122 | |
|---|
| .. | .. |
|---|
| 187 | 168 | #ifndef smp_load_acquire |
|---|
| 188 | 169 | #define smp_load_acquire(p) \ |
|---|
| 189 | 170 | ({ \ |
|---|
| 190 | | - typeof(*p) ___p1 = READ_ONCE(*p); \ |
|---|
| 171 | + __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ |
|---|
| 191 | 172 | compiletime_assert_atomic_type(*p); \ |
|---|
| 192 | 173 | barrier(); \ |
|---|
| 193 | | - ___p1; \ |
|---|
| 174 | + (typeof(*p))___p1; \ |
|---|
| 194 | 175 | }) |
|---|
| 195 | 176 | #endif |
|---|
| 196 | 177 | |
|---|
| .. | .. |
|---|
| 200 | 181 | #define virt_mb() __smp_mb() |
|---|
| 201 | 182 | #define virt_rmb() __smp_rmb() |
|---|
| 202 | 183 | #define virt_wmb() __smp_wmb() |
|---|
| 203 | | -#define virt_read_barrier_depends() __smp_read_barrier_depends() |
|---|
| 204 | 184 | #define virt_store_mb(var, value) __smp_store_mb(var, value) |
|---|
| 205 | 185 | #define virt_mb__before_atomic() __smp_mb__before_atomic() |
|---|
| 206 | 186 | #define virt_mb__after_atomic() __smp_mb__after_atomic() |
|---|
| .. | .. |
|---|
| 233 | 213 | #ifndef smp_cond_load_relaxed |
|---|
| 234 | 214 | #define smp_cond_load_relaxed(ptr, cond_expr) ({ \ |
|---|
| 235 | 215 | typeof(ptr) __PTR = (ptr); \ |
|---|
| 236 | | - typeof(*ptr) VAL; \ |
|---|
| 216 | + __unqual_scalar_typeof(*ptr) VAL; \ |
|---|
| 237 | 217 | for (;;) { \ |
|---|
| 238 | 218 | VAL = READ_ONCE(*__PTR); \ |
|---|
| 239 | 219 | if (cond_expr) \ |
|---|
| 240 | 220 | break; \ |
|---|
| 241 | 221 | cpu_relax(); \ |
|---|
| 242 | 222 | } \ |
|---|
| 243 | | - VAL; \ |
|---|
| 223 | + (typeof(*ptr))VAL; \ |
|---|
| 244 | 224 | }) |
|---|
| 245 | 225 | #endif |
|---|
| 246 | 226 | |
|---|
| .. | .. |
|---|
| 254 | 234 | */ |
|---|
| 255 | 235 | #ifndef smp_cond_load_acquire |
|---|
| 256 | 236 | #define smp_cond_load_acquire(ptr, cond_expr) ({ \ |
|---|
| 257 | | - typeof(*ptr) _val; \ |
|---|
| 237 | + __unqual_scalar_typeof(*ptr) _val; \ |
|---|
| 258 | 238 | _val = smp_cond_load_relaxed(ptr, cond_expr); \ |
|---|
| 259 | 239 | smp_acquire__after_ctrl_dep(); \ |
|---|
| 260 | | - _val; \ |
|---|
| 240 | + (typeof(*ptr))_val; \ |
|---|
| 261 | 241 | }) |
|---|
| 262 | 242 | #endif |
|---|
| 263 | 243 | |
|---|
| 244 | +/* |
|---|
| 245 | + * pmem_wmb() ensures that all stores for which the modification |
|---|
| 246 | + * are written to persistent storage by preceding instructions have |
|---|
| 247 | + * updated persistent storage before any data access or data transfer |
|---|
| 248 | + * caused by subsequent instructions is initiated. |
|---|
| 249 | + */ |
|---|
| 250 | +#ifndef pmem_wmb |
|---|
| 251 | +#define pmem_wmb() wmb() |
|---|
| 252 | +#endif |
|---|
| 253 | + |
|---|
| 264 | 254 | #endif /* !__ASSEMBLY__ */ |
|---|
| 265 | 255 | #endif /* __ASM_GENERIC_BARRIER_H */ |
|---|