.. | .. |
---|
| 1 | +/* SPDX-License-Identifier: GPL-2.0-or-later */ |
---|
1 | 2 | /* |
---|
2 | 3 | * Generic barrier definitions. |
---|
3 | 4 | * |
---|
.. | .. |
---|
6 | 7 | * |
---|
7 | 8 | * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. |
---|
8 | 9 | * Written by David Howells (dhowells@redhat.com) |
---|
9 | | - * |
---|
10 | | - * This program is free software; you can redistribute it and/or |
---|
11 | | - * modify it under the terms of the GNU General Public Licence |
---|
12 | | - * as published by the Free Software Foundation; either version |
---|
13 | | - * 2 of the Licence, or (at your option) any later version. |
---|
14 | 10 | */ |
---|
15 | 11 | #ifndef __ASM_GENERIC_BARRIER_H |
---|
16 | 12 | #define __ASM_GENERIC_BARRIER_H |
---|
.. | .. |
---|
18 | 14 | #ifndef __ASSEMBLY__ |
---|
19 | 15 | |
---|
20 | 16 | #include <linux/compiler.h> |
---|
| 17 | +#include <asm/rwonce.h> |
---|
21 | 18 | |
---|
22 | 19 | #ifndef nop |
---|
23 | 20 | #define nop() asm volatile ("nop") |
---|
.. | .. |
---|
50 | 47 | #define dma_wmb() wmb() |
---|
51 | 48 | #endif |
---|
52 | 49 | |
---|
53 | | -#ifndef read_barrier_depends |
---|
54 | | -#define read_barrier_depends() do { } while (0) |
---|
55 | | -#endif |
---|
56 | | - |
---|
57 | 50 | #ifndef __smp_mb |
---|
58 | 51 | #define __smp_mb() mb() |
---|
59 | 52 | #endif |
---|
.. | .. |
---|
64 | 57 | |
---|
65 | 58 | #ifndef __smp_wmb |
---|
66 | 59 | #define __smp_wmb() wmb() |
---|
67 | | -#endif |
---|
68 | | - |
---|
69 | | -#ifndef __smp_read_barrier_depends |
---|
70 | | -#define __smp_read_barrier_depends() read_barrier_depends() |
---|
71 | 60 | #endif |
---|
72 | 61 | |
---|
73 | 62 | #ifdef CONFIG_SMP |
---|
.. | .. |
---|
84 | 73 | #define smp_wmb() __smp_wmb() |
---|
85 | 74 | #endif |
---|
86 | 75 | |
---|
87 | | -#ifndef smp_read_barrier_depends |
---|
88 | | -#define smp_read_barrier_depends() __smp_read_barrier_depends() |
---|
89 | | -#endif |
---|
90 | | - |
---|
91 | 76 | #else /* !CONFIG_SMP */ |
---|
92 | 77 | |
---|
93 | 78 | #ifndef smp_mb |
---|
.. | .. |
---|
100 | 85 | |
---|
101 | 86 | #ifndef smp_wmb |
---|
102 | 87 | #define smp_wmb() barrier() |
---|
103 | | -#endif |
---|
104 | | - |
---|
105 | | -#ifndef smp_read_barrier_depends |
---|
106 | | -#define smp_read_barrier_depends() do { } while (0) |
---|
107 | 88 | #endif |
---|
108 | 89 | |
---|
109 | 90 | #endif /* CONFIG_SMP */ |
---|
.. | .. |
---|
132 | 113 | #ifndef __smp_load_acquire |
---|
133 | 114 | #define __smp_load_acquire(p) \ |
---|
134 | 115 | ({ \ |
---|
135 | | - typeof(*p) ___p1 = READ_ONCE(*p); \ |
---|
| 116 | + __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ |
---|
136 | 117 | compiletime_assert_atomic_type(*p); \ |
---|
137 | 118 | __smp_mb(); \ |
---|
138 | | - ___p1; \ |
---|
| 119 | + (typeof(*p))___p1; \ |
---|
139 | 120 | }) |
---|
140 | 121 | #endif |
---|
141 | 122 | |
---|
.. | .. |
---|
187 | 168 | #ifndef smp_load_acquire |
---|
188 | 169 | #define smp_load_acquire(p) \ |
---|
189 | 170 | ({ \ |
---|
190 | | - typeof(*p) ___p1 = READ_ONCE(*p); \ |
---|
| 171 | + __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ |
---|
191 | 172 | compiletime_assert_atomic_type(*p); \ |
---|
192 | 173 | barrier(); \ |
---|
193 | | - ___p1; \ |
---|
| 174 | + (typeof(*p))___p1; \ |
---|
194 | 175 | }) |
---|
195 | 176 | #endif |
---|
196 | 177 | |
---|
.. | .. |
---|
200 | 181 | #define virt_mb() __smp_mb() |
---|
201 | 182 | #define virt_rmb() __smp_rmb() |
---|
202 | 183 | #define virt_wmb() __smp_wmb() |
---|
203 | | -#define virt_read_barrier_depends() __smp_read_barrier_depends() |
---|
204 | 184 | #define virt_store_mb(var, value) __smp_store_mb(var, value) |
---|
205 | 185 | #define virt_mb__before_atomic() __smp_mb__before_atomic() |
---|
206 | 186 | #define virt_mb__after_atomic() __smp_mb__after_atomic() |
---|
.. | .. |
---|
233 | 213 | #ifndef smp_cond_load_relaxed |
---|
234 | 214 | #define smp_cond_load_relaxed(ptr, cond_expr) ({ \ |
---|
235 | 215 | typeof(ptr) __PTR = (ptr); \ |
---|
236 | | - typeof(*ptr) VAL; \ |
---|
| 216 | + __unqual_scalar_typeof(*ptr) VAL; \ |
---|
237 | 217 | for (;;) { \ |
---|
238 | 218 | VAL = READ_ONCE(*__PTR); \ |
---|
239 | 219 | if (cond_expr) \ |
---|
240 | 220 | break; \ |
---|
241 | 221 | cpu_relax(); \ |
---|
242 | 222 | } \ |
---|
243 | | - VAL; \ |
---|
| 223 | + (typeof(*ptr))VAL; \ |
---|
244 | 224 | }) |
---|
245 | 225 | #endif |
---|
246 | 226 | |
---|
.. | .. |
---|
254 | 234 | */ |
---|
255 | 235 | #ifndef smp_cond_load_acquire |
---|
256 | 236 | #define smp_cond_load_acquire(ptr, cond_expr) ({ \ |
---|
257 | | - typeof(*ptr) _val; \ |
---|
| 237 | + __unqual_scalar_typeof(*ptr) _val; \ |
---|
258 | 238 | _val = smp_cond_load_relaxed(ptr, cond_expr); \ |
---|
259 | 239 | smp_acquire__after_ctrl_dep(); \ |
---|
260 | | - _val; \ |
---|
| 240 | + (typeof(*ptr))_val; \ |
---|
261 | 241 | }) |
---|
262 | 242 | #endif |
---|
263 | 243 | |
---|
| 244 | +/* |
---|
| 245 | + * pmem_wmb() ensures that all stores for which the modification |
---|
| 246 | + * are written to persistent storage by preceding instructions have |
---|
| 247 | + * updated persistent storage before any data access or data transfer |
---|
| 248 | + * caused by subsequent instructions is initiated. |
---|
| 249 | + */ |
---|
| 250 | +#ifndef pmem_wmb |
---|
| 251 | +#define pmem_wmb() wmb() |
---|
| 252 | +#endif |
---|
| 253 | + |
---|
264 | 254 | #endif /* !__ASSEMBLY__ */ |
---|
265 | 255 | #endif /* __ASM_GENERIC_BARRIER_H */ |
---|