add ax88772_rst

hc

2024-05-10 9999e48639b3cecb08ffb37358bcba3b48161b29

 kernel/arch/x86/include/asm/percpu.h
..
..
@@ -4,33 +4,15 @@
4
4
 
5
5
 #ifdef CONFIG_X86_64
6
6
 #define __percpu_seg		gs
7
-#define __percpu_mov_op		movq
8
7
 #else
9
8
 #define __percpu_seg		fs
10
-#define __percpu_mov_op		movl
11
9
 #endif
12
10
 
13
11
 #ifdef __ASSEMBLY__
14
12
 
15
-/*
16
- * PER_CPU finds an address of a per-cpu variable.
17
- *
18
- * Args:
19
- *    var - variable name
20
- *    reg - 32bit register
21
- *
22
- * The resulting address is stored in the "reg" argument.
23
- *
24
- * Example:
25
- *    PER_CPU(cpu_gdt_descr, %ebx)
26
- */
27
13
 #ifdef CONFIG_SMP
28
-#define PER_CPU(var, reg)						\
29
-	__percpu_mov_op %__percpu_seg:this_cpu_off, reg;		\
30
-	lea var(reg), reg
31
14
 #define PER_CPU_VAR(var)	%__percpu_seg:var
32
15
 #else /* ! SMP */
33
-#define PER_CPU(var, reg)	__percpu_mov_op $var, reg
34
16
 #define PER_CPU_VAR(var)	var
35
17
 #endif	/* SMP */
36
18
 
..
..
@@ -85,213 +67,108 @@
85
67
 
86
68
 /* For arch-specific code, we can use direct single-insn ops (they
87
69
  * don't give an lvalue though). */
88
-extern void __bad_percpu_size(void);
89
70
 
90
-#define percpu_to_op(op, var, val)			\
91
-do {							\
92
-	typedef typeof(var) pto_T__;			\
93
-	if (0) {					\
94
-		pto_T__ pto_tmp__;			\
95
-		pto_tmp__ = (val);			\
96
-		(void)pto_tmp__;			\
97
-	}						\
98
-	switch (sizeof(var)) {				\
99
-	case 1:						\
100
-		asm(op "b %1,"__percpu_arg(0)		\
101
-		    : "+m" (var)			\
102
-		    : "qi" ((pto_T__)(val)));		\
103
-		break;					\
104
-	case 2:						\
105
-		asm(op "w %1,"__percpu_arg(0)		\
106
-		    : "+m" (var)			\
107
-		    : "ri" ((pto_T__)(val)));		\
108
-		break;					\
109
-	case 4:						\
110
-		asm(op "l %1,"__percpu_arg(0)		\
111
-		    : "+m" (var)			\
112
-		    : "ri" ((pto_T__)(val)));		\
113
-		break;					\
114
-	case 8:						\
115
-		asm(op "q %1,"__percpu_arg(0)		\
116
-		    : "+m" (var)			\
117
-		    : "re" ((pto_T__)(val)));		\
118
-		break;					\
119
-	default: __bad_percpu_size();			\
120
-	}						\
71
+#define __pcpu_type_1 u8
72
+#define __pcpu_type_2 u16
73
+#define __pcpu_type_4 u32
74
+#define __pcpu_type_8 u64
75
+
76
+#define __pcpu_cast_1(val) ((u8)(((unsigned long) val) & 0xff))
77
+#define __pcpu_cast_2(val) ((u16)(((unsigned long) val) & 0xffff))
78
+#define __pcpu_cast_4(val) ((u32)(((unsigned long) val) & 0xffffffff))
79
+#define __pcpu_cast_8(val) ((u64)(val))
80
+
81
+#define __pcpu_op1_1(op, dst) op "b " dst
82
+#define __pcpu_op1_2(op, dst) op "w " dst
83
+#define __pcpu_op1_4(op, dst) op "l " dst
84
+#define __pcpu_op1_8(op, dst) op "q " dst
85
+
86
+#define __pcpu_op2_1(op, src, dst) op "b " src ", " dst
87
+#define __pcpu_op2_2(op, src, dst) op "w " src ", " dst
88
+#define __pcpu_op2_4(op, src, dst) op "l " src ", " dst
89
+#define __pcpu_op2_8(op, src, dst) op "q " src ", " dst
90
+
91
+#define __pcpu_reg_1(mod, x) mod "q" (x)
92
+#define __pcpu_reg_2(mod, x) mod "r" (x)
93
+#define __pcpu_reg_4(mod, x) mod "r" (x)
94
+#define __pcpu_reg_8(mod, x) mod "r" (x)
95
+
96
+#define __pcpu_reg_imm_1(x) "qi" (x)
97
+#define __pcpu_reg_imm_2(x) "ri" (x)
98
+#define __pcpu_reg_imm_4(x) "ri" (x)
99
+#define __pcpu_reg_imm_8(x) "re" (x)
100
+
101
+#define percpu_to_op(size, qual, op, _var, _val)			\
102
+do {									\
103
+	__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val);	\
104
+	if (0) {		                                        \
105
+		typeof(_var) pto_tmp__;					\
106
+		pto_tmp__ = (_val);					\
107
+		(void)pto_tmp__;					\
108
+	}								\
109
+	asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var]))	\
110
+	    : [var] "+m" (_var)						\
111
+	    : [val] __pcpu_reg_imm_##size(pto_val__));			\
121
112
 } while (0)
113
+
114
+#define percpu_unary_op(size, qual, op, _var)				\
115
+({									\
116
+	asm qual (__pcpu_op1_##size(op, __percpu_arg([var]))		\
117
+	    : [var] "+m" (_var));					\
118
+})
122
119
 
123
120
 /*
124
121
  * Generate a percpu add to memory instruction and optimize code
125
122
  * if one is added or subtracted.
126
123
  */
127
-#define percpu_add_op(var, val)						\
124
+#define percpu_add_op(size, qual, var, val)				\
128
125
 do {									\
129
-	typedef typeof(var) pao_T__;					\
130
126
 	const int pao_ID__ = (__builtin_constant_p(val) &&		\
131
127
 			      ((val) == 1 || (val) == -1)) ?		\
132
128
 				(int)(val) : 0;				\
133
129
 	if (0) {							\
134
-		pao_T__ pao_tmp__;					\
130
+		typeof(var) pao_tmp__;					\
135
131
 		pao_tmp__ = (val);					\
136
132
 		(void)pao_tmp__;					\
137
133
 	}								\
138
-	switch (sizeof(var)) {						\
139
-	case 1:								\
140
-		if (pao_ID__ == 1)					\
141
-			asm("incb "__percpu_arg(0) : "+m" (var));	\
142
-		else if (pao_ID__ == -1)				\
143
-			asm("decb "__percpu_arg(0) : "+m" (var));	\
144
-		else							\
145
-			asm("addb %1, "__percpu_arg(0)			\
146
-			    : "+m" (var)				\
147
-			    : "qi" ((pao_T__)(val)));			\
148
-		break;							\
149
-	case 2:								\
150
-		if (pao_ID__ == 1)					\
151
-			asm("incw "__percpu_arg(0) : "+m" (var));	\
152
-		else if (pao_ID__ == -1)				\
153
-			asm("decw "__percpu_arg(0) : "+m" (var));	\
154
-		else							\
155
-			asm("addw %1, "__percpu_arg(0)			\
156
-			    : "+m" (var)				\
157
-			    : "ri" ((pao_T__)(val)));			\
158
-		break;							\
159
-	case 4:								\
160
-		if (pao_ID__ == 1)					\
161
-			asm("incl "__percpu_arg(0) : "+m" (var));	\
162
-		else if (pao_ID__ == -1)				\
163
-			asm("decl "__percpu_arg(0) : "+m" (var));	\
164
-		else							\
165
-			asm("addl %1, "__percpu_arg(0)			\
166
-			    : "+m" (var)				\
167
-			    : "ri" ((pao_T__)(val)));			\
168
-		break;							\
169
-	case 8:								\
170
-		if (pao_ID__ == 1)					\
171
-			asm("incq "__percpu_arg(0) : "+m" (var));	\
172
-		else if (pao_ID__ == -1)				\
173
-			asm("decq "__percpu_arg(0) : "+m" (var));	\
174
-		else							\
175
-			asm("addq %1, "__percpu_arg(0)			\
176
-			    : "+m" (var)				\
177
-			    : "re" ((pao_T__)(val)));			\
178
-		break;							\
179
-	default: __bad_percpu_size();					\
180
-	}								\
134
+	if (pao_ID__ == 1)						\
135
+		percpu_unary_op(size, qual, "inc", var);		\
136
+	else if (pao_ID__ == -1)					\
137
+		percpu_unary_op(size, qual, "dec", var);		\
138
+	else								\
139
+		percpu_to_op(size, qual, "add", var, val);		\
181
140
 } while (0)
182
141
 
183
-#define percpu_from_op(op, var)				\
184
-({							\
185
-	typeof(var) pfo_ret__;				\
186
-	switch (sizeof(var)) {				\
187
-	case 1:						\
188
-		asm volatile(op "b "__percpu_arg(1)",%0"\
189
-		    : "=q" (pfo_ret__)			\
190
-		    : "m" (var));			\
191
-		break;					\
192
-	case 2:						\
193
-		asm volatile(op "w "__percpu_arg(1)",%0"\
194
-		    : "=r" (pfo_ret__)			\
195
-		    : "m" (var));			\
196
-		break;					\
197
-	case 4:						\
198
-		asm volatile(op "l "__percpu_arg(1)",%0"\
199
-		    : "=r" (pfo_ret__)			\
200
-		    : "m" (var));			\
201
-		break;					\
202
-	case 8:						\
203
-		asm volatile(op "q "__percpu_arg(1)",%0"\
204
-		    : "=r" (pfo_ret__)			\
205
-		    : "m" (var));			\
206
-		break;					\
207
-	default: __bad_percpu_size();			\
208
-	}						\
209
-	pfo_ret__;					\
142
+#define percpu_from_op(size, qual, op, _var)				\
143
+({									\
144
+	__pcpu_type_##size pfo_val__;					\
145
+	asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]")	\
146
+	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
147
+	    : [var] "m" (_var));					\
148
+	(typeof(_var))(unsigned long) pfo_val__;			\
210
149
 })
211
150
 
212
-#define percpu_stable_op(op, var)			\
213
-({							\
214
-	typeof(var) pfo_ret__;				\
215
-	switch (sizeof(var)) {				\
216
-	case 1:						\
217
-		asm(op "b "__percpu_arg(P1)",%0"	\
218
-		    : "=q" (pfo_ret__)			\
219
-		    : "p" (&(var)));			\
220
-		break;					\
221
-	case 2:						\
222
-		asm(op "w "__percpu_arg(P1)",%0"	\
223
-		    : "=r" (pfo_ret__)			\
224
-		    : "p" (&(var)));			\
225
-		break;					\
226
-	case 4:						\
227
-		asm(op "l "__percpu_arg(P1)",%0"	\
228
-		    : "=r" (pfo_ret__)			\
229
-		    : "p" (&(var)));			\
230
-		break;					\
231
-	case 8:						\
232
-		asm(op "q "__percpu_arg(P1)",%0"	\
233
-		    : "=r" (pfo_ret__)			\
234
-		    : "p" (&(var)));			\
235
-		break;					\
236
-	default: __bad_percpu_size();			\
237
-	}						\
238
-	pfo_ret__;					\
239
-})
240
-
241
-#define percpu_unary_op(op, var)			\
242
-({							\
243
-	switch (sizeof(var)) {				\
244
-	case 1:						\
245
-		asm(op "b "__percpu_arg(0)		\
246
-		    : "+m" (var));			\
247
-		break;					\
248
-	case 2:						\
249
-		asm(op "w "__percpu_arg(0)		\
250
-		    : "+m" (var));			\
251
-		break;					\
252
-	case 4:						\
253
-		asm(op "l "__percpu_arg(0)		\
254
-		    : "+m" (var));			\
255
-		break;					\
256
-	case 8:						\
257
-		asm(op "q "__percpu_arg(0)		\
258
-		    : "+m" (var));			\
259
-		break;					\
260
-	default: __bad_percpu_size();			\
261
-	}						\
151
+#define percpu_stable_op(size, op, _var)				\
152
+({									\
153
+	__pcpu_type_##size pfo_val__;					\
154
+	asm(__pcpu_op2_##size(op, __percpu_arg(P[var]), "%[val]")	\
155
+	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
156
+	    : [var] "p" (&(_var)));					\
157
+	(typeof(_var))(unsigned long) pfo_val__;			\
262
158
 })
263
159
 
264
160
 /*
265
161
  * Add return operation
266
162
  */
267
-#define percpu_add_return_op(var, val)					\
163
+#define percpu_add_return_op(size, qual, _var, _val)			\
268
164
 ({									\
269
-	typeof(var) paro_ret__ = val;					\
270
-	switch (sizeof(var)) {						\
271
-	case 1:								\
272
-		asm("xaddb %0, "__percpu_arg(1)				\
273
-			    : "+q" (paro_ret__), "+m" (var)		\
274
-			    : : "memory");				\
275
-		break;							\
276
-	case 2:								\
277
-		asm("xaddw %0, "__percpu_arg(1)				\
278
-			    : "+r" (paro_ret__), "+m" (var)		\
279
-			    : : "memory");				\
280
-		break;							\
281
-	case 4:								\
282
-		asm("xaddl %0, "__percpu_arg(1)				\
283
-			    : "+r" (paro_ret__), "+m" (var)		\
284
-			    : : "memory");				\
285
-		break;							\
286
-	case 8:								\
287
-		asm("xaddq %0, "__percpu_arg(1)				\
288
-			    : "+re" (paro_ret__), "+m" (var)		\
289
-			    : : "memory");				\
290
-		break;							\
291
-	default: __bad_percpu_size();					\
292
-	}								\
293
-	paro_ret__ += val;						\
294
-	paro_ret__;							\
165
+	__pcpu_type_##size paro_tmp__ = __pcpu_cast_##size(_val);	\
166
+	asm qual (__pcpu_op2_##size("xadd", "%[tmp]",			\
167
+				     __percpu_arg([var]))		\
168
+		  : [tmp] __pcpu_reg_##size("+", paro_tmp__),		\
169
+		    [var] "+m" (_var)					\
170
+		  : : "memory");					\
171
+	(typeof(_var))(unsigned long) (paro_tmp__ + _val);		\
295
172
 })
296
173
 
297
174
 /*
..
..
@@ -299,85 +176,38 @@
299
176
  * expensive due to the implied lock prefix.  The processor cannot prefetch
300
177
  * cachelines if xchg is used.
301
178
  */
302
-#define percpu_xchg_op(var, nval)					\
179
+#define percpu_xchg_op(size, qual, _var, _nval)				\
303
180
 ({									\
304
-	typeof(var) pxo_ret__;						\
305
-	typeof(var) pxo_new__ = (nval);					\
306
-	switch (sizeof(var)) {						\
307
-	case 1:								\
308
-		asm("\n\tmov "__percpu_arg(1)",%%al"			\
309
-		    "\n1:\tcmpxchgb %2, "__percpu_arg(1)		\
310
-		    "\n\tjnz 1b"					\
311
-			    : "=&a" (pxo_ret__), "+m" (var)		\
312
-			    : "q" (pxo_new__)				\
313
-			    : "memory");				\
314
-		break;							\
315
-	case 2:								\
316
-		asm("\n\tmov "__percpu_arg(1)",%%ax"			\
317
-		    "\n1:\tcmpxchgw %2, "__percpu_arg(1)		\
318
-		    "\n\tjnz 1b"					\
319
-			    : "=&a" (pxo_ret__), "+m" (var)		\
320
-			    : "r" (pxo_new__)				\
321
-			    : "memory");				\
322
-		break;							\
323
-	case 4:								\
324
-		asm("\n\tmov "__percpu_arg(1)",%%eax"			\
325
-		    "\n1:\tcmpxchgl %2, "__percpu_arg(1)		\
326
-		    "\n\tjnz 1b"					\
327
-			    : "=&a" (pxo_ret__), "+m" (var)		\
328
-			    : "r" (pxo_new__)				\
329
-			    : "memory");				\
330
-		break;							\
331
-	case 8:								\
332
-		asm("\n\tmov "__percpu_arg(1)",%%rax"			\
333
-		    "\n1:\tcmpxchgq %2, "__percpu_arg(1)		\
334
-		    "\n\tjnz 1b"					\
335
-			    : "=&a" (pxo_ret__), "+m" (var)		\
336
-			    : "r" (pxo_new__)				\
337
-			    : "memory");				\
338
-		break;							\
339
-	default: __bad_percpu_size();					\
340
-	}								\
341
-	pxo_ret__;							\
181
+	__pcpu_type_##size pxo_old__;					\
182
+	__pcpu_type_##size pxo_new__ = __pcpu_cast_##size(_nval);	\
183
+	asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]),		\
184
+				    "%[oval]")				\
185
+		  "\n1:\t"						\
186
+		  __pcpu_op2_##size("cmpxchg", "%[nval]",		\
187
+				    __percpu_arg([var]))		\
188
+		  "\n\tjnz 1b"						\
189
+		  : [oval] "=&a" (pxo_old__),				\
190
+		    [var] "+m" (_var)					\
191
+		  : [nval] __pcpu_reg_##size(, pxo_new__)		\
192
+		  : "memory");						\
193
+	(typeof(_var))(unsigned long) pxo_old__;			\
342
194
 })
343
195
 
344
196
 /*
345
197
  * cmpxchg has no such implied lock semantics as a result it is much
346
198
  * more efficient for cpu local operations.
347
199
  */
348
-#define percpu_cmpxchg_op(var, oval, nval)				\
200
+#define percpu_cmpxchg_op(size, qual, _var, _oval, _nval)		\
349
201
 ({									\
350
-	typeof(var) pco_ret__;						\
351
-	typeof(var) pco_old__ = (oval);					\
352
-	typeof(var) pco_new__ = (nval);					\
353
-	switch (sizeof(var)) {						\
354
-	case 1:								\
355
-		asm("cmpxchgb %2, "__percpu_arg(1)			\
356
-			    : "=a" (pco_ret__), "+m" (var)		\
357
-			    : "q" (pco_new__), "0" (pco_old__)		\
358
-			    : "memory");				\
359
-		break;							\
360
-	case 2:								\
361
-		asm("cmpxchgw %2, "__percpu_arg(1)			\
362
-			    : "=a" (pco_ret__), "+m" (var)		\
363
-			    : "r" (pco_new__), "0" (pco_old__)		\
364
-			    : "memory");				\
365
-		break;							\
366
-	case 4:								\
367
-		asm("cmpxchgl %2, "__percpu_arg(1)			\
368
-			    : "=a" (pco_ret__), "+m" (var)		\
369
-			    : "r" (pco_new__), "0" (pco_old__)		\
370
-			    : "memory");				\
371
-		break;							\
372
-	case 8:								\
373
-		asm("cmpxchgq %2, "__percpu_arg(1)			\
374
-			    : "=a" (pco_ret__), "+m" (var)		\
375
-			    : "r" (pco_new__), "0" (pco_old__)		\
376
-			    : "memory");				\
377
-		break;							\
378
-	default: __bad_percpu_size();					\
379
-	}								\
380
-	pco_ret__;							\
202
+	__pcpu_type_##size pco_old__ = __pcpu_cast_##size(_oval);	\
203
+	__pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval);	\
204
+	asm qual (__pcpu_op2_##size("cmpxchg", "%[nval]",		\
205
+				    __percpu_arg([var]))		\
206
+		  : [oval] "+a" (pco_old__),				\
207
+		    [var] "+m" (_var)					\
208
+		  : [nval] __pcpu_reg_##size(, pco_new__)		\
209
+		  : "memory");						\
210
+	(typeof(_var))(unsigned long) pco_old__;			\
381
211
 })
382
212
 
383
213
 /*
..
..
@@ -389,60 +219,76 @@
389
219
  * per-thread variables implemented as per-cpu variables and thus
390
220
  * stable for the duration of the respective task.
391
221
  */
392
-#define this_cpu_read_stable(var)	percpu_stable_op("mov", var)
222
+#define this_cpu_read_stable_1(pcp)	percpu_stable_op(1, "mov", pcp)
223
+#define this_cpu_read_stable_2(pcp)	percpu_stable_op(2, "mov", pcp)
224
+#define this_cpu_read_stable_4(pcp)	percpu_stable_op(4, "mov", pcp)
225
+#define this_cpu_read_stable_8(pcp)	percpu_stable_op(8, "mov", pcp)
226
+#define this_cpu_read_stable(pcp)	__pcpu_size_call_return(this_cpu_read_stable_, pcp)
393
227
 
394
-#define raw_cpu_read_1(pcp)		percpu_from_op("mov", pcp)
395
-#define raw_cpu_read_2(pcp)		percpu_from_op("mov", pcp)
396
-#define raw_cpu_read_4(pcp)		percpu_from_op("mov", pcp)
228
+#define raw_cpu_read_1(pcp)		percpu_from_op(1, , "mov", pcp)
229
+#define raw_cpu_read_2(pcp)		percpu_from_op(2, , "mov", pcp)
230
+#define raw_cpu_read_4(pcp)		percpu_from_op(4, , "mov", pcp)
397
231
 
398
-#define raw_cpu_write_1(pcp, val)	percpu_to_op("mov", (pcp), val)
399
-#define raw_cpu_write_2(pcp, val)	percpu_to_op("mov", (pcp), val)
400
-#define raw_cpu_write_4(pcp, val)	percpu_to_op("mov", (pcp), val)
401
-#define raw_cpu_add_1(pcp, val)		percpu_add_op((pcp), val)
402
-#define raw_cpu_add_2(pcp, val)		percpu_add_op((pcp), val)
403
-#define raw_cpu_add_4(pcp, val)		percpu_add_op((pcp), val)
404
-#define raw_cpu_and_1(pcp, val)		percpu_to_op("and", (pcp), val)
405
-#define raw_cpu_and_2(pcp, val)		percpu_to_op("and", (pcp), val)
406
-#define raw_cpu_and_4(pcp, val)		percpu_to_op("and", (pcp), val)
407
-#define raw_cpu_or_1(pcp, val)		percpu_to_op("or", (pcp), val)
408
-#define raw_cpu_or_2(pcp, val)		percpu_to_op("or", (pcp), val)
409
-#define raw_cpu_or_4(pcp, val)		percpu_to_op("or", (pcp), val)
410
-#define raw_cpu_xchg_1(pcp, val)	percpu_xchg_op(pcp, val)
411
-#define raw_cpu_xchg_2(pcp, val)	percpu_xchg_op(pcp, val)
412
-#define raw_cpu_xchg_4(pcp, val)	percpu_xchg_op(pcp, val)
232
+#define raw_cpu_write_1(pcp, val)	percpu_to_op(1, , "mov", (pcp), val)
233
+#define raw_cpu_write_2(pcp, val)	percpu_to_op(2, , "mov", (pcp), val)
234
+#define raw_cpu_write_4(pcp, val)	percpu_to_op(4, , "mov", (pcp), val)
235
+#define raw_cpu_add_1(pcp, val)		percpu_add_op(1, , (pcp), val)
236
+#define raw_cpu_add_2(pcp, val)		percpu_add_op(2, , (pcp), val)
237
+#define raw_cpu_add_4(pcp, val)		percpu_add_op(4, , (pcp), val)
238
+#define raw_cpu_and_1(pcp, val)		percpu_to_op(1, , "and", (pcp), val)
239
+#define raw_cpu_and_2(pcp, val)		percpu_to_op(2, , "and", (pcp), val)
240
+#define raw_cpu_and_4(pcp, val)		percpu_to_op(4, , "and", (pcp), val)
241
+#define raw_cpu_or_1(pcp, val)		percpu_to_op(1, , "or", (pcp), val)
242
+#define raw_cpu_or_2(pcp, val)		percpu_to_op(2, , "or", (pcp), val)
243
+#define raw_cpu_or_4(pcp, val)		percpu_to_op(4, , "or", (pcp), val)
413
244
 
414
-#define this_cpu_read_1(pcp)		percpu_from_op("mov", pcp)
415
-#define this_cpu_read_2(pcp)		percpu_from_op("mov", pcp)
416
-#define this_cpu_read_4(pcp)		percpu_from_op("mov", pcp)
417
-#define this_cpu_write_1(pcp, val)	percpu_to_op("mov", (pcp), val)
418
-#define this_cpu_write_2(pcp, val)	percpu_to_op("mov", (pcp), val)
419
-#define this_cpu_write_4(pcp, val)	percpu_to_op("mov", (pcp), val)
420
-#define this_cpu_add_1(pcp, val)	percpu_add_op((pcp), val)
421
-#define this_cpu_add_2(pcp, val)	percpu_add_op((pcp), val)
422
-#define this_cpu_add_4(pcp, val)	percpu_add_op((pcp), val)
423
-#define this_cpu_and_1(pcp, val)	percpu_to_op("and", (pcp), val)
424
-#define this_cpu_and_2(pcp, val)	percpu_to_op("and", (pcp), val)
425
-#define this_cpu_and_4(pcp, val)	percpu_to_op("and", (pcp), val)
426
-#define this_cpu_or_1(pcp, val)		percpu_to_op("or", (pcp), val)
427
-#define this_cpu_or_2(pcp, val)		percpu_to_op("or", (pcp), val)
428
-#define this_cpu_or_4(pcp, val)		percpu_to_op("or", (pcp), val)
429
-#define this_cpu_xchg_1(pcp, nval)	percpu_xchg_op(pcp, nval)
430
-#define this_cpu_xchg_2(pcp, nval)	percpu_xchg_op(pcp, nval)
431
-#define this_cpu_xchg_4(pcp, nval)	percpu_xchg_op(pcp, nval)
245
+/*
246
+ * raw_cpu_xchg() can use a load-store since it is not required to be
247
+ * IRQ-safe.
248
+ */
249
+#define raw_percpu_xchg_op(var, nval)					\
250
+({									\
251
+	typeof(var) pxo_ret__ = raw_cpu_read(var);			\
252
+	raw_cpu_write(var, (nval));					\
253
+	pxo_ret__;							\
254
+})
432
255
 
433
-#define raw_cpu_add_return_1(pcp, val)		percpu_add_return_op(pcp, val)
434
-#define raw_cpu_add_return_2(pcp, val)		percpu_add_return_op(pcp, val)
435
-#define raw_cpu_add_return_4(pcp, val)		percpu_add_return_op(pcp, val)
436
-#define raw_cpu_cmpxchg_1(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
437
-#define raw_cpu_cmpxchg_2(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
438
-#define raw_cpu_cmpxchg_4(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
256
+#define raw_cpu_xchg_1(pcp, val)	raw_percpu_xchg_op(pcp, val)
257
+#define raw_cpu_xchg_2(pcp, val)	raw_percpu_xchg_op(pcp, val)
258
+#define raw_cpu_xchg_4(pcp, val)	raw_percpu_xchg_op(pcp, val)
439
259
 
440
-#define this_cpu_add_return_1(pcp, val)		percpu_add_return_op(pcp, val)
441
-#define this_cpu_add_return_2(pcp, val)		percpu_add_return_op(pcp, val)
442
-#define this_cpu_add_return_4(pcp, val)		percpu_add_return_op(pcp, val)
443
-#define this_cpu_cmpxchg_1(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
444
-#define this_cpu_cmpxchg_2(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
445
-#define this_cpu_cmpxchg_4(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
260
+#define this_cpu_read_1(pcp)		percpu_from_op(1, volatile, "mov", pcp)
261
+#define this_cpu_read_2(pcp)		percpu_from_op(2, volatile, "mov", pcp)
262
+#define this_cpu_read_4(pcp)		percpu_from_op(4, volatile, "mov", pcp)
263
+#define this_cpu_write_1(pcp, val)	percpu_to_op(1, volatile, "mov", (pcp), val)
264
+#define this_cpu_write_2(pcp, val)	percpu_to_op(2, volatile, "mov", (pcp), val)
265
+#define this_cpu_write_4(pcp, val)	percpu_to_op(4, volatile, "mov", (pcp), val)
266
+#define this_cpu_add_1(pcp, val)	percpu_add_op(1, volatile, (pcp), val)
267
+#define this_cpu_add_2(pcp, val)	percpu_add_op(2, volatile, (pcp), val)
268
+#define this_cpu_add_4(pcp, val)	percpu_add_op(4, volatile, (pcp), val)
269
+#define this_cpu_and_1(pcp, val)	percpu_to_op(1, volatile, "and", (pcp), val)
270
+#define this_cpu_and_2(pcp, val)	percpu_to_op(2, volatile, "and", (pcp), val)
271
+#define this_cpu_and_4(pcp, val)	percpu_to_op(4, volatile, "and", (pcp), val)
272
+#define this_cpu_or_1(pcp, val)		percpu_to_op(1, volatile, "or", (pcp), val)
273
+#define this_cpu_or_2(pcp, val)		percpu_to_op(2, volatile, "or", (pcp), val)
274
+#define this_cpu_or_4(pcp, val)		percpu_to_op(4, volatile, "or", (pcp), val)
275
+#define this_cpu_xchg_1(pcp, nval)	percpu_xchg_op(1, volatile, pcp, nval)
276
+#define this_cpu_xchg_2(pcp, nval)	percpu_xchg_op(2, volatile, pcp, nval)
277
+#define this_cpu_xchg_4(pcp, nval)	percpu_xchg_op(4, volatile, pcp, nval)
278
+
279
+#define raw_cpu_add_return_1(pcp, val)		percpu_add_return_op(1, , pcp, val)
280
+#define raw_cpu_add_return_2(pcp, val)		percpu_add_return_op(2, , pcp, val)
281
+#define raw_cpu_add_return_4(pcp, val)		percpu_add_return_op(4, , pcp, val)
282
+#define raw_cpu_cmpxchg_1(pcp, oval, nval)	percpu_cmpxchg_op(1, , pcp, oval, nval)
283
+#define raw_cpu_cmpxchg_2(pcp, oval, nval)	percpu_cmpxchg_op(2, , pcp, oval, nval)
284
+#define raw_cpu_cmpxchg_4(pcp, oval, nval)	percpu_cmpxchg_op(4, , pcp, oval, nval)
285
+
286
+#define this_cpu_add_return_1(pcp, val)		percpu_add_return_op(1, volatile, pcp, val)
287
+#define this_cpu_add_return_2(pcp, val)		percpu_add_return_op(2, volatile, pcp, val)
288
+#define this_cpu_add_return_4(pcp, val)		percpu_add_return_op(4, volatile, pcp, val)
289
+#define this_cpu_cmpxchg_1(pcp, oval, nval)	percpu_cmpxchg_op(1, volatile, pcp, oval, nval)
290
+#define this_cpu_cmpxchg_2(pcp, oval, nval)	percpu_cmpxchg_op(2, volatile, pcp, oval, nval)
291
+#define this_cpu_cmpxchg_4(pcp, oval, nval)	percpu_cmpxchg_op(4, volatile, pcp, oval, nval)
446
292
 
447
293
 #ifdef CONFIG_X86_CMPXCHG64
448
294
 #define percpu_cmpxchg8b_double(pcp1, pcp2, o1, o2, n1, n2)		\
..
..
@@ -466,23 +312,23 @@
466
312
  * 32 bit must fall back to generic operations.
467
313
  */
468
314
 #ifdef CONFIG_X86_64
469
-#define raw_cpu_read_8(pcp)			percpu_from_op("mov", pcp)
470
-#define raw_cpu_write_8(pcp, val)		percpu_to_op("mov", (pcp), val)
471
-#define raw_cpu_add_8(pcp, val)			percpu_add_op((pcp), val)
472
-#define raw_cpu_and_8(pcp, val)			percpu_to_op("and", (pcp), val)
473
-#define raw_cpu_or_8(pcp, val)			percpu_to_op("or", (pcp), val)
474
-#define raw_cpu_add_return_8(pcp, val)		percpu_add_return_op(pcp, val)
475
-#define raw_cpu_xchg_8(pcp, nval)		percpu_xchg_op(pcp, nval)
476
-#define raw_cpu_cmpxchg_8(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
315
+#define raw_cpu_read_8(pcp)			percpu_from_op(8, , "mov", pcp)
316
+#define raw_cpu_write_8(pcp, val)		percpu_to_op(8, , "mov", (pcp), val)
317
+#define raw_cpu_add_8(pcp, val)			percpu_add_op(8, , (pcp), val)
318
+#define raw_cpu_and_8(pcp, val)			percpu_to_op(8, , "and", (pcp), val)
319
+#define raw_cpu_or_8(pcp, val)			percpu_to_op(8, , "or", (pcp), val)
320
+#define raw_cpu_add_return_8(pcp, val)		percpu_add_return_op(8, , pcp, val)
321
+#define raw_cpu_xchg_8(pcp, nval)		raw_percpu_xchg_op(pcp, nval)
322
+#define raw_cpu_cmpxchg_8(pcp, oval, nval)	percpu_cmpxchg_op(8, , pcp, oval, nval)
477
323
 
478
-#define this_cpu_read_8(pcp)			percpu_from_op("mov", pcp)
479
-#define this_cpu_write_8(pcp, val)		percpu_to_op("mov", (pcp), val)
480
-#define this_cpu_add_8(pcp, val)		percpu_add_op((pcp), val)
481
-#define this_cpu_and_8(pcp, val)		percpu_to_op("and", (pcp), val)
482
-#define this_cpu_or_8(pcp, val)			percpu_to_op("or", (pcp), val)
483
-#define this_cpu_add_return_8(pcp, val)		percpu_add_return_op(pcp, val)
484
-#define this_cpu_xchg_8(pcp, nval)		percpu_xchg_op(pcp, nval)
485
-#define this_cpu_cmpxchg_8(pcp, oval, nval)	percpu_cmpxchg_op(pcp, oval, nval)
324
+#define this_cpu_read_8(pcp)			percpu_from_op(8, volatile, "mov", pcp)
325
+#define this_cpu_write_8(pcp, val)		percpu_to_op(8, volatile, "mov", (pcp), val)
326
+#define this_cpu_add_8(pcp, val)		percpu_add_op(8, volatile, (pcp), val)
327
+#define this_cpu_and_8(pcp, val)		percpu_to_op(8, volatile, "and", (pcp), val)
328
+#define this_cpu_or_8(pcp, val)			percpu_to_op(8, volatile, "or", (pcp), val)
329
+#define this_cpu_add_return_8(pcp, val)		percpu_add_return_op(8, volatile, pcp, val)
330
+#define this_cpu_xchg_8(pcp, nval)		percpu_xchg_op(8, volatile, pcp, nval)
331
+#define this_cpu_cmpxchg_8(pcp, oval, nval)	percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
486
332
 
487
333
 /*
488
334
  * Pretty complex macro to generate cmpxchg16 instruction.  The instruction