.. | .. |
---|
12 | 12 | #include <asm/processor.h> |
---|
13 | 13 | #include <asm/smap.h> |
---|
14 | 14 | |
---|
15 | | -#define __futex_atomic_op1(insn, ret, oldval, uaddr, oparg) \ |
---|
16 | | - asm volatile("\t" ASM_STAC "\n" \ |
---|
17 | | - "1:\t" insn "\n" \ |
---|
18 | | - "2:\t" ASM_CLAC "\n" \ |
---|
| 15 | +#define unsafe_atomic_op1(insn, oval, uaddr, oparg, label) \ |
---|
| 16 | +do { \ |
---|
| 17 | + int oldval = 0, ret; \ |
---|
| 18 | + asm volatile("1:\t" insn "\n" \ |
---|
| 19 | + "2:\n" \ |
---|
19 | 20 | "\t.section .fixup,\"ax\"\n" \ |
---|
20 | 21 | "3:\tmov\t%3, %1\n" \ |
---|
21 | 22 | "\tjmp\t2b\n" \ |
---|
22 | 23 | "\t.previous\n" \ |
---|
23 | | - _ASM_EXTABLE(1b, 3b) \ |
---|
| 24 | + _ASM_EXTABLE_UA(1b, 3b) \ |
---|
24 | 25 | : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \ |
---|
25 | | - : "i" (-EFAULT), "0" (oparg), "1" (0)) |
---|
| 26 | + : "i" (-EFAULT), "0" (oparg), "1" (0)); \ |
---|
| 27 | + if (ret) \ |
---|
| 28 | + goto label; \ |
---|
| 29 | + *oval = oldval; \ |
---|
| 30 | +} while(0) |
---|
26 | 31 | |
---|
27 | | -#define __futex_atomic_op2(insn, ret, oldval, uaddr, oparg) \ |
---|
28 | | - asm volatile("\t" ASM_STAC "\n" \ |
---|
29 | | - "1:\tmovl %2, %0\n" \ |
---|
30 | | - "\tmovl\t%0, %3\n" \ |
---|
| 32 | + |
---|
| 33 | +#define unsafe_atomic_op2(insn, oval, uaddr, oparg, label) \ |
---|
| 34 | +do { \ |
---|
| 35 | + int oldval = 0, ret, tem; \ |
---|
| 36 | + asm volatile("1:\tmovl %2, %0\n" \ |
---|
| 37 | + "2:\tmovl\t%0, %3\n" \ |
---|
31 | 38 | "\t" insn "\n" \ |
---|
32 | | - "2:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \ |
---|
33 | | - "\tjnz\t1b\n" \ |
---|
34 | | - "3:\t" ASM_CLAC "\n" \ |
---|
| 39 | + "3:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \ |
---|
| 40 | + "\tjnz\t2b\n" \ |
---|
| 41 | + "4:\n" \ |
---|
35 | 42 | "\t.section .fixup,\"ax\"\n" \ |
---|
36 | | - "4:\tmov\t%5, %1\n" \ |
---|
37 | | - "\tjmp\t3b\n" \ |
---|
| 43 | + "5:\tmov\t%5, %1\n" \ |
---|
| 44 | + "\tjmp\t4b\n" \ |
---|
38 | 45 | "\t.previous\n" \ |
---|
39 | | - _ASM_EXTABLE(1b, 4b) \ |
---|
40 | | - _ASM_EXTABLE(2b, 4b) \ |
---|
| 46 | + _ASM_EXTABLE_UA(1b, 5b) \ |
---|
| 47 | + _ASM_EXTABLE_UA(3b, 5b) \ |
---|
41 | 48 | : "=&a" (oldval), "=&r" (ret), \ |
---|
42 | 49 | "+m" (*uaddr), "=&r" (tem) \ |
---|
43 | | - : "r" (oparg), "i" (-EFAULT), "1" (0)) |
---|
| 50 | + : "r" (oparg), "i" (-EFAULT), "1" (0)); \ |
---|
| 51 | + if (ret) \ |
---|
| 52 | + goto label; \ |
---|
| 53 | + *oval = oldval; \ |
---|
| 54 | +} while(0) |
---|
44 | 55 | |
---|
45 | | -static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, |
---|
| 56 | +static __always_inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, |
---|
46 | 57 | u32 __user *uaddr) |
---|
47 | 58 | { |
---|
48 | | - int oldval = 0, ret, tem; |
---|
49 | | - |
---|
50 | | - pagefault_disable(); |
---|
| 59 | + if (!user_access_begin(uaddr, sizeof(u32))) |
---|
| 60 | + return -EFAULT; |
---|
51 | 61 | |
---|
52 | 62 | switch (op) { |
---|
53 | 63 | case FUTEX_OP_SET: |
---|
54 | | - __futex_atomic_op1("xchgl %0, %2", ret, oldval, uaddr, oparg); |
---|
| 64 | + unsafe_atomic_op1("xchgl %0, %2", oval, uaddr, oparg, Efault); |
---|
55 | 65 | break; |
---|
56 | 66 | case FUTEX_OP_ADD: |
---|
57 | | - __futex_atomic_op1(LOCK_PREFIX "xaddl %0, %2", ret, oldval, |
---|
58 | | - uaddr, oparg); |
---|
| 67 | + unsafe_atomic_op1(LOCK_PREFIX "xaddl %0, %2", oval, |
---|
| 68 | + uaddr, oparg, Efault); |
---|
59 | 69 | break; |
---|
60 | 70 | case FUTEX_OP_OR: |
---|
61 | | - __futex_atomic_op2("orl %4, %3", ret, oldval, uaddr, oparg); |
---|
| 71 | + unsafe_atomic_op2("orl %4, %3", oval, uaddr, oparg, Efault); |
---|
62 | 72 | break; |
---|
63 | 73 | case FUTEX_OP_ANDN: |
---|
64 | | - __futex_atomic_op2("andl %4, %3", ret, oldval, uaddr, ~oparg); |
---|
| 74 | + unsafe_atomic_op2("andl %4, %3", oval, uaddr, ~oparg, Efault); |
---|
65 | 75 | break; |
---|
66 | 76 | case FUTEX_OP_XOR: |
---|
67 | | - __futex_atomic_op2("xorl %4, %3", ret, oldval, uaddr, oparg); |
---|
| 77 | + unsafe_atomic_op2("xorl %4, %3", oval, uaddr, oparg, Efault); |
---|
68 | 78 | break; |
---|
69 | 79 | default: |
---|
70 | | - ret = -ENOSYS; |
---|
| 80 | + user_access_end(); |
---|
| 81 | + return -ENOSYS; |
---|
71 | 82 | } |
---|
72 | | - |
---|
73 | | - pagefault_enable(); |
---|
74 | | - |
---|
75 | | - if (!ret) |
---|
76 | | - *oval = oldval; |
---|
77 | | - |
---|
78 | | - return ret; |
---|
| 83 | + user_access_end(); |
---|
| 84 | + return 0; |
---|
| 85 | +Efault: |
---|
| 86 | + user_access_end(); |
---|
| 87 | + return -EFAULT; |
---|
79 | 88 | } |
---|
80 | 89 | |
---|
81 | 90 | static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, |
---|
82 | 91 | u32 oldval, u32 newval) |
---|
83 | 92 | { |
---|
84 | | - return user_atomic_cmpxchg_inatomic(uval, uaddr, oldval, newval); |
---|
| 93 | + int ret = 0; |
---|
| 94 | + |
---|
| 95 | + if (!user_access_begin(uaddr, sizeof(u32))) |
---|
| 96 | + return -EFAULT; |
---|
| 97 | + asm volatile("\n" |
---|
| 98 | + "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" |
---|
| 99 | + "2:\n" |
---|
| 100 | + "\t.section .fixup, \"ax\"\n" |
---|
| 101 | + "3:\tmov %3, %0\n" |
---|
| 102 | + "\tjmp 2b\n" |
---|
| 103 | + "\t.previous\n" |
---|
| 104 | + _ASM_EXTABLE_UA(1b, 3b) |
---|
| 105 | + : "+r" (ret), "=a" (oldval), "+m" (*uaddr) |
---|
| 106 | + : "i" (-EFAULT), "r" (newval), "1" (oldval) |
---|
| 107 | + : "memory" |
---|
| 108 | + ); |
---|
| 109 | + user_access_end(); |
---|
| 110 | + *uval = oldval; |
---|
| 111 | + return ret; |
---|
85 | 112 | } |
---|
86 | 113 | |
---|
87 | 114 | #endif |
---|