.. | .. |
---|
14 | 14 | #define wmb() asm volatile("dmb ishst" ::: "memory") |
---|
15 | 15 | #define rmb() asm volatile("dmb ishld" ::: "memory") |
---|
16 | 16 | |
---|
| 17 | +/* |
---|
| 18 | + * Kernel uses dmb variants on arm64 for smp_*() barriers. Pretty much the same |
---|
| 19 | + * implementation as above mb()/wmb()/rmb(), though for the latter kernel uses |
---|
| 20 | + * dsb. In any case, should above mb()/wmb()/rmb() change, make sure the below |
---|
| 21 | + * smp_*() don't. |
---|
| 22 | + */ |
---|
| 23 | +#define smp_mb() asm volatile("dmb ish" ::: "memory") |
---|
| 24 | +#define smp_wmb() asm volatile("dmb ishst" ::: "memory") |
---|
| 25 | +#define smp_rmb() asm volatile("dmb ishld" ::: "memory") |
---|
| 26 | + |
---|
| 27 | +#define smp_store_release(p, v) \ |
---|
| 28 | +do { \ |
---|
| 29 | + union { typeof(*p) __val; char __c[1]; } __u = \ |
---|
| 30 | + { .__val = (v) }; \ |
---|
| 31 | + \ |
---|
| 32 | + switch (sizeof(*p)) { \ |
---|
| 33 | + case 1: \ |
---|
| 34 | + asm volatile ("stlrb %w1, %0" \ |
---|
| 35 | + : "=Q" (*p) \ |
---|
| 36 | + : "r" (*(__u8_alias_t *)__u.__c) \ |
---|
| 37 | + : "memory"); \ |
---|
| 38 | + break; \ |
---|
| 39 | + case 2: \ |
---|
| 40 | + asm volatile ("stlrh %w1, %0" \ |
---|
| 41 | + : "=Q" (*p) \ |
---|
| 42 | + : "r" (*(__u16_alias_t *)__u.__c) \ |
---|
| 43 | + : "memory"); \ |
---|
| 44 | + break; \ |
---|
| 45 | + case 4: \ |
---|
| 46 | + asm volatile ("stlr %w1, %0" \ |
---|
| 47 | + : "=Q" (*p) \ |
---|
| 48 | + : "r" (*(__u32_alias_t *)__u.__c) \ |
---|
| 49 | + : "memory"); \ |
---|
| 50 | + break; \ |
---|
| 51 | + case 8: \ |
---|
| 52 | + asm volatile ("stlr %1, %0" \ |
---|
| 53 | + : "=Q" (*p) \ |
---|
| 54 | + : "r" (*(__u64_alias_t *)__u.__c) \ |
---|
| 55 | + : "memory"); \ |
---|
| 56 | + break; \ |
---|
| 57 | + default: \ |
---|
| 58 | + /* Only to shut up gcc ... */ \ |
---|
| 59 | + mb(); \ |
---|
| 60 | + break; \ |
---|
| 61 | + } \ |
---|
| 62 | +} while (0) |
---|
| 63 | + |
---|
| 64 | +#define smp_load_acquire(p) \ |
---|
| 65 | +({ \ |
---|
| 66 | + union { typeof(*p) __val; char __c[1]; } __u = \ |
---|
| 67 | + { .__c = { 0 } }; \ |
---|
| 68 | + \ |
---|
| 69 | + switch (sizeof(*p)) { \ |
---|
| 70 | + case 1: \ |
---|
| 71 | + asm volatile ("ldarb %w0, %1" \ |
---|
| 72 | + : "=r" (*(__u8_alias_t *)__u.__c) \ |
---|
| 73 | + : "Q" (*p) : "memory"); \ |
---|
| 74 | + break; \ |
---|
| 75 | + case 2: \ |
---|
| 76 | + asm volatile ("ldarh %w0, %1" \ |
---|
| 77 | + : "=r" (*(__u16_alias_t *)__u.__c) \ |
---|
| 78 | + : "Q" (*p) : "memory"); \ |
---|
| 79 | + break; \ |
---|
| 80 | + case 4: \ |
---|
| 81 | + asm volatile ("ldar %w0, %1" \ |
---|
| 82 | + : "=r" (*(__u32_alias_t *)__u.__c) \ |
---|
| 83 | + : "Q" (*p) : "memory"); \ |
---|
| 84 | + break; \ |
---|
| 85 | + case 8: \ |
---|
| 86 | + asm volatile ("ldar %0, %1" \ |
---|
| 87 | + : "=r" (*(__u64_alias_t *)__u.__c) \ |
---|
| 88 | + : "Q" (*p) : "memory"); \ |
---|
| 89 | + break; \ |
---|
| 90 | + default: \ |
---|
| 91 | + /* Only to shut up gcc ... */ \ |
---|
| 92 | + mb(); \ |
---|
| 93 | + break; \ |
---|
| 94 | + } \ |
---|
| 95 | + __u.__val; \ |
---|
| 96 | +}) |
---|
| 97 | + |
---|
17 | 98 | #endif /* _TOOLS_LINUX_ASM_AARCH64_BARRIER_H */ |
---|