~hc/RK356X_SDK_RELEASE.git

..	..	@@ -52,16 +52,14 @@
52	52	STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
53	53
54	54	_TIF_WORK = (_TIF_SIGPENDING \| _TIF_NOTIFY_RESUME \| _TIF_NEED_RESCHED \| \
55		- _TIF_UPROBE \| _TIF_GUARDED_STORAGE \| _TIF_PATCH_PENDING)
	55	+ _TIF_UPROBE \| _TIF_GUARDED_STORAGE \| _TIF_PATCH_PENDING \| \
	56	+ _TIF_NOTIFY_SIGNAL)
56	57	_TIF_TRACE = (_TIF_SYSCALL_TRACE \| _TIF_SYSCALL_AUDIT \| _TIF_SECCOMP \| \
57	58	_TIF_SYSCALL_TRACEPOINT)
58		-_CIF_WORK = (_CIF_MCCK_PENDING \| _CIF_ASCE_PRIMARY \| \
59		- _CIF_ASCE_SECONDARY \| _CIF_FPU)
	59	+_CIF_WORK = (_CIF_ASCE_PRIMARY \| _CIF_ASCE_SECONDARY \| _CIF_FPU)
60	60	_PIF_WORK = (_PIF_PER_TRAP \| _PIF_SYSCALL_RESTART)
61	61
62	62	_LPP_OFFSET = __LC_LPP
63		-
64		-#define BASED(name) name-cleanup_critical(%r13)
65	63
66	64	.macro TRACE_IRQS_ON
67	65	#ifdef CONFIG_TRACE_IRQFLAGS
..	..	@@ -85,36 +83,97 @@
85	83	#endif
86	84	.endm
87	85
88		- .macro CHECK_STACK stacksize,savearea
	86	+ .macro CHECK_STACK savearea
89	87	#ifdef CONFIG_CHECK_STACK
90		- tml %r15,\stacksize - CONFIG_STACK_GUARD
	88	+ tml %r15,STACK_SIZE - CONFIG_STACK_GUARD
91	89	lghi %r14,\savearea
92	90	jz stack_overflow
93	91	#endif
94	92	.endm
95	93
96		- .macro SWITCH_ASYNC savearea,timer
	94	+ .macro CHECK_VMAP_STACK savearea,oklabel
	95	+#ifdef CONFIG_VMAP_STACK
	96	+ lgr %r14,%r15
	97	+ nill %r14,0x10000 - STACK_SIZE
	98	+ oill %r14,STACK_INIT
	99	+ clg %r14,__LC_KERNEL_STACK
	100	+ je \oklabel
	101	+ clg %r14,__LC_ASYNC_STACK
	102	+ je \oklabel
	103	+ clg %r14,__LC_NODAT_STACK
	104	+ je \oklabel
	105	+ clg %r14,__LC_RESTART_STACK
	106	+ je \oklabel
	107	+ lghi %r14,\savearea
	108	+ j stack_overflow
	109	+#else
	110	+ j \oklabel
	111	+#endif
	112	+ .endm
	113	+
	114	+ .macro SWITCH_ASYNC savearea,timer,clock
97	115	tmhh %r8,0x0001 # interrupting from user ?
98		- jnz 1f
	116	+ jnz 4f
	117	+#if IS_ENABLED(CONFIG_KVM)
99	118	lgr %r14,%r9
100		- slg %r14,BASED(.Lcritical_start)
101		- clg %r14,BASED(.Lcritical_length)
	119	+ larl %r13,.Lsie_gmap
	120	+ slgr %r14,%r13
	121	+ lghi %r13,.Lsie_done - .Lsie_gmap
	122	+ clgr %r14,%r13
102	123	jhe 0f
103	124	lghi %r11,\savearea # inside critical section, do cleanup
104		- brasl %r14,cleanup_critical
105		- tmhh %r8,0x0001 # retest problem state after cleanup
106		- jnz 1f
107		-0: lg %r14,__LC_ASYNC_STACK # are we already on the async stack?
	125	+ brasl %r14,.Lcleanup_sie
	126	+#endif
	127	+0: larl %r13,.Lpsw_idle_exit
	128	+ cgr %r13,%r9
	129	+ jne 3f
	130	+
	131	+ larl %r1,smp_cpu_mtid
	132	+ llgf %r1,0(%r1)
	133	+ ltgr %r1,%r1
	134	+ jz 2f # no SMT, skip mt_cycles calculation
	135	+ .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15)
	136	+ larl %r3,mt_cycles
	137	+ ag %r3,__LC_PERCPU_OFFSET
	138	+ la %r4,__SF_EMPTY+16(%r15)
	139	+1: lg %r0,0(%r3)
	140	+ slg %r0,0(%r4)
	141	+ alg %r0,64(%r4)
	142	+ stg %r0,0(%r3)
	143	+ la %r3,8(%r3)
	144	+ la %r4,8(%r4)
	145	+ brct %r1,1b
	146	+
	147	+2: mvc __CLOCK_IDLE_EXIT(8,%r2), \clock
	148	+ mvc __TIMER_IDLE_EXIT(8,%r2), \timer
	149	+ # account system time going idle
	150	+ ni __LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT
	151	+
	152	+ lg %r13,__LC_STEAL_TIMER
	153	+ alg %r13,__CLOCK_IDLE_ENTER(%r2)
	154	+ slg %r13,__LC_LAST_UPDATE_CLOCK
	155	+ stg %r13,__LC_STEAL_TIMER
	156	+
	157	+ mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
	158	+
	159	+ lg %r13,__LC_SYSTEM_TIMER
	160	+ alg %r13,__LC_LAST_UPDATE_TIMER
	161	+ slg %r13,__TIMER_IDLE_ENTER(%r2)
	162	+ stg %r13,__LC_SYSTEM_TIMER
	163	+ mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
	164	+
	165	+ nihh %r8,0xfcfd # clear wait state and irq bits
	166	+3: lg %r14,__LC_ASYNC_STACK # are we already on the target stack?
108	167	slgr %r14,%r15
109	168	srag %r14,%r14,STACK_SHIFT
110		- jnz 2f
111		- CHECK_STACK 1<<STACK_SHIFT,\savearea
	169	+ jnz 5f
	170	+ CHECK_STACK \savearea
112	171	aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
113		- j 3f
114		-1: UPDATE_VTIME %r14,%r15,\timer
	172	+ j 6f
	173	+4: UPDATE_VTIME %r14,%r15,\timer
115	174	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
116		-2: lg %r15,__LC_ASYNC_STACK # load async stack
117		-3: la %r11,STACK_FRAME_OVERHEAD(%r15)
	175	+5: lg %r15,__LC_ASYNC_STACK # load async stack
	176	+6: la %r11,STACK_FRAME_OVERHEAD(%r15)
118	177	.endm
119	178
120	179	.macro UPDATE_VTIME w1,w2,enter_timer
..	..	@@ -129,10 +188,28 @@
129	188	mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
130	189	.endm
131	190
132		- .macro REENABLE_IRQS
	191	+ .macro RESTORE_SM_CLEAR_PER
133	192	stg %r8,__LC_RETURN_PSW
134	193	ni __LC_RETURN_PSW,0xbf
135	194	ssm __LC_RETURN_PSW
	195	+ .endm
	196	+
	197	+ .macro ENABLE_INTS
	198	+ stosm __SF_EMPTY(%r15),3
	199	+ .endm
	200	+
	201	+ .macro ENABLE_INTS_TRACE
	202	+ TRACE_IRQS_ON
	203	+ ENABLE_INTS
	204	+ .endm
	205	+
	206	+ .macro DISABLE_INTS
	207	+ stnsm __SF_EMPTY(%r15),0xfc
	208	+ .endm
	209	+
	210	+ .macro DISABLE_INTS_TRACE
	211	+ DISABLE_INTS
	212	+ TRACE_IRQS_OFF
136	213	.endm
137	214
138	215	.macro STCK savearea
..	..	@@ -204,6 +281,7 @@
204	281	.globl __bpon
205	282	BPON
206	283	BR_EX %r14
	284	+ENDPROC(__bpon)
207	285
208	286	/*
209	287	* Scheduler resume function, called by switch_to
..	..	@@ -216,10 +294,10 @@
216	294	stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
217	295	lghi %r4,__TASK_stack
218	296	lghi %r1,__TASK_thread
219		- lg %r5,0(%r4,%r3) # start of kernel stack of next
	297	+ llill %r5,STACK_INIT
220	298	stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev
221		- lgr %r15,%r5
222		- aghi %r15,STACK_INIT # end of kernel stack of next
	299	+ lg %r15,0(%r4,%r3) # start of kernel stack of next
	300	+ agr %r15,%r5 # end of kernel stack of next
223	301	stg %r3,__LC_CURRENT # store task struct of next
224	302	stg %r15,__LC_KERNEL_STACK # store end of kernel stack
225	303	lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next
..	..	@@ -228,8 +306,7 @@
228	306	lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
229	307	ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
230	308	BR_EX %r14
231		-
232		-.L__critical_start:
	309	+ENDPROC(__switch_to)
233	310
234	311	#if IS_ENABLED(CONFIG_KVM)
235	312	/*
..	..	@@ -263,7 +340,6 @@
263	340	BPEXIT __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP\|_TIF_ISOLATE_BP_GUEST)
264	341	.Lsie_entry:
265	342	sie 0(%r14)
266		-.Lsie_exit:
267	343	BPOFF
268	344	BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP\|_TIF_ISOLATE_BP_GUEST)
269	345	.Lsie_skip:
..	..	@@ -304,27 +380,26 @@
304	380	EX_TABLE(.Lrewind_pad4,.Lsie_fault)
305	381	EX_TABLE(.Lrewind_pad2,.Lsie_fault)
306	382	EX_TABLE(sie_exit,.Lsie_fault)
	383	+ENDPROC(sie64a)
307	384	EXPORT_SYMBOL(sie64a)
308	385	EXPORT_SYMBOL(sie_exit)
309	386	#endif
310	387
311	388	/*
312	389	* SVC interrupt handler routine. System calls are synchronous events and
313		- * are executed with interrupts enabled.
	390	+ * are entered with interrupts disabled.
314	391	*/
315	392
316	393	ENTRY(system_call)
317	394	stpt __LC_SYNC_ENTER_TIMER
318		-.Lsysc_stmg:
319	395	stmg %r8,%r15,__LC_SAVE_AREA_SYNC
320	396	BPOFF
321	397	lg %r12,__LC_CURRENT
322		- lghi %r13,__TASK_thread
323	398	lghi %r14,_PIF_SYSCALL
324	399	.Lsysc_per:
	400	+ lghi %r13,__TASK_thread
325	401	lg %r15,__LC_KERNEL_STACK
326	402	la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
327		-.Lsysc_vtime:
328	403	UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
329	404	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
330	405	stmg %r0,%r7,__PT_R0(%r11)
..	..	@@ -332,25 +407,26 @@
332	407	mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW
333	408	mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
334	409	stg %r14,__PT_FLAGS(%r11)
	410	+ xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	411	+ ENABLE_INTS
335	412	.Lsysc_do_svc:
336	413	# clear user controlled register to prevent speculative use
337	414	xgr %r0,%r0
338	415	# load address of system call table
339	416	lg %r10,__THREAD_sysc_table(%r13,%r12)
340	417	llgh %r8,__PT_INT_CODE+2(%r11)
341		- slag %r8,%r8,2 # shift and test for svc 0
	418	+ slag %r8,%r8,3 # shift and test for svc 0
342	419	jnz .Lsysc_nr_ok
343	420	# svc 0: system call number in %r1
344	421	llgfr %r1,%r1 # clear high word in r1
	422	+ sth %r1,__PT_INT_CODE+2(%r11)
345	423	cghi %r1,NR_syscalls
346	424	jnl .Lsysc_nr_ok
347		- sth %r1,__PT_INT_CODE+2(%r11)
348		- slag %r8,%r1,2
	425	+ slag %r8,%r1,3
349	426	.Lsysc_nr_ok:
350		- xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
351	427	stg %r2,__PT_ORIG_GPR2(%r11)
352	428	stg %r7,STACK_FRAME_OVERHEAD(%r15)
353		- lgf %r9,0(%r8,%r10) # get system call add.
	429	+ lg %r9,0(%r8,%r10) # get system call add.
354	430	TSTMSK __TI_flags(%r12),_TIF_TRACE
355	431	jnz .Lsysc_tracesys
356	432	BASR_EX %r14,%r9 # call sys_xxxx
..	..	@@ -363,30 +439,30 @@
363	439	#endif
364	440	LOCKDEP_SYS_EXIT
365	441	.Lsysc_tif:
	442	+ DISABLE_INTS
366	443	TSTMSK __PT_FLAGS(%r11),_PIF_WORK
367	444	jnz .Lsysc_work
368	445	TSTMSK __TI_flags(%r12),_TIF_WORK
369	446	jnz .Lsysc_work # check for work
370		- TSTMSK __LC_CPU_FLAGS,_CIF_WORK
	447	+ TSTMSK __LC_CPU_FLAGS,(_CIF_WORK-_CIF_FPU)
371	448	jnz .Lsysc_work
372	449	BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
373	450	.Lsysc_restore:
374		- lg %r14,__LC_VDSO_PER_CPU
375		- lmg %r0,%r10,__PT_R0(%r11)
	451	+ DISABLE_INTS
	452	+ TSTMSK __LC_CPU_FLAGS, _CIF_FPU
	453	+ jz .Lsysc_skip_fpu
	454	+ brasl %r14,load_fpu_regs
	455	+.Lsysc_skip_fpu:
376	456	mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
377		-.Lsysc_exit_timer:
378	457	stpt __LC_EXIT_TIMER
379		- mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
380		- lmg %r11,%r15,__PT_R11(%r11)
381		- lpswe __LC_RETURN_PSW
382		-.Lsysc_done:
	458	+ lmg %r0,%r15,__PT_R0(%r11)
	459	+ b __LC_RETURN_LPSWE
383	460
384	461	#
385	462	# One of the work bits is on. Find out which one.
386	463	#
387	464	.Lsysc_work:
388		- TSTMSK __LC_CPU_FLAGS,_CIF_MCCK_PENDING
389		- jo .Lsysc_mcck_pending
	465	+ ENABLE_INTS
390	466	TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED
391	467	jo .Lsysc_reschedule
392	468	TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART
..	..	@@ -406,15 +482,13 @@
406	482	#endif
407	483	TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL_RESTART
408	484	jo .Lsysc_syscall_restart
409		- TSTMSK __TI_flags(%r12),_TIF_SIGPENDING
410		- jo .Lsysc_sigpending
	485	+ TSTMSK __TI_flags(%r12),(_TIF_SIGPENDING\|_TIF_NOTIFY_SIGNAL)
	486	+ jnz .Lsysc_sigpending
411	487	TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME
412	488	jo .Lsysc_notify_resume
413		- TSTMSK __LC_CPU_FLAGS,_CIF_FPU
414		- jo .Lsysc_vxrs
415	489	TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY\|_CIF_ASCE_SECONDARY)
416	490	jnz .Lsysc_asce
417		- j .Lsysc_return # beware of critical section cleanup
	491	+ j .Lsysc_return
418	492
419	493	#
420	494	# _TIF_NEED_RESCHED is set, call schedule
..	..	@@ -422,13 +496,6 @@
422	496	.Lsysc_reschedule:
423	497	larl %r14,.Lsysc_return
424	498	jg schedule
425		-
426		-#
427		-# _CIF_MCCK_PENDING is set, call handler
428		-#
429		-.Lsysc_mcck_pending:
430		- larl %r14,.Lsysc_return
431		- jg s390_handle_mcck # TIF bit will be cleared by handler
432	499
433	500	#
434	501	# _CIF_ASCE_PRIMARY and/or _CIF_ASCE_SECONDARY set, load user space asce
..	..	@@ -449,12 +516,6 @@
449	516	larl %r14,.Lsysc_return
450	517	jg set_fs_fixup
451	518
452		-#
453		-# CIF_FPU is set, restore floating-point controls and floating-point registers.
454		-#
455		-.Lsysc_vxrs:
456		- larl %r14,.Lsysc_return
457		- jg load_fpu_regs
458	519
459	520	#
460	521	# _TIF_SIGPENDING is set, call do_signal
..	..	@@ -536,9 +597,8 @@
536	597	lghi %r0,NR_syscalls
537	598	clgr %r0,%r2
538	599	jnh .Lsysc_tracenogo
539		- sllg %r8,%r2,2
540		- lgf %r9,0(%r8,%r10)
541		-.Lsysc_tracego:
	600	+ sllg %r8,%r2,3
	601	+ lg %r9,0(%r8,%r10)
542	602	lmg %r3,%r7,__PT_R3(%r11)
543	603	stg %r7,STACK_FRAME_OVERHEAD(%r15)
544	604	lg %r2,__PT_ORIG_GPR2(%r11)
..	..	@@ -550,6 +610,7 @@
550	610	lgr %r2,%r11 # pass pointer to pt_regs
551	611	larl %r14,.Lsysc_return
552	612	jg do_syscall_trace_exit
	613	+ENDPROC(system_call)
553	614
554	615	#
555	616	# a new process exits the kernel with ret_from_fork
..	..	@@ -558,16 +619,20 @@
558	619	la %r11,STACK_FRAME_OVERHEAD(%r15)
559	620	lg %r12,__LC_CURRENT
560	621	brasl %r14,schedule_tail
561		- TRACE_IRQS_ON
562		- ssm __LC_SVC_NEW_PSW # reenable interrupts
563	622	tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
564	623	jne .Lsysc_tracenogo
565	624	# it's a kernel thread
566	625	lmg %r9,%r10,__PT_R9(%r11) # load gprs
	626	+ la %r2,0(%r10)
	627	+ BASR_EX %r14,%r9
	628	+ j .Lsysc_tracenogo
	629	+ENDPROC(ret_from_fork)
	630	+
567	631	ENTRY(kernel_thread_starter)
568	632	la %r2,0(%r10)
569	633	BASR_EX %r14,%r9
570	634	j .Lsysc_tracenogo
	635	+ENDPROC(kernel_thread_starter)
571	636
572	637	/*
573	638	* Program check handler routine
..	..	@@ -578,42 +643,51 @@
578	643	BPOFF
579	644	stmg %r8,%r15,__LC_SAVE_AREA_SYNC
580	645	lg %r10,__LC_LAST_BREAK
581		- lg %r12,__LC_CURRENT
	646	+ srag %r11,%r10,12
	647	+ jnz 0f
	648	+ /* if __LC_LAST_BREAK is < 4096, it contains one of
	649	+ * the lpswe addresses in lowcore. Set it to 1 (initial state)
	650	+ * to prevent leaking that address to userspace.
	651	+ */
	652	+ lghi %r10,1
	653	+0: lg %r12,__LC_CURRENT
582	654	lghi %r11,0
583		- larl %r13,cleanup_critical
584	655	lmg %r8,%r9,__LC_PGM_OLD_PSW
585	656	tmhh %r8,0x0001 # test problem state bit
586		- jnz 2f # -> fault in user space
	657	+ jnz 3f # -> fault in user space
587	658	#if IS_ENABLED(CONFIG_KVM)
588	659	# cleanup critical section for program checks in sie64a
589	660	lgr %r14,%r9
590		- slg %r14,BASED(.Lsie_critical_start)
591		- clg %r14,BASED(.Lsie_critical_length)
592		- jhe 0f
	661	+ larl %r13,.Lsie_gmap
	662	+ slgr %r14,%r13
	663	+ lghi %r13,.Lsie_done - .Lsie_gmap
	664	+ clgr %r14,%r13
	665	+ jhe 1f
593	666	lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
594	667	ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
595	668	lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
596	669	larl %r9,sie_exit # skip forward to sie_exit
597	670	lghi %r11,_PIF_GUEST_FAULT
598	671	#endif
599		-0: tmhh %r8,0x4000 # PER bit set in old PSW ?
600		- jnz 1f # -> enabled, can't be a double fault
	672	+1: tmhh %r8,0x4000 # PER bit set in old PSW ?
	673	+ jnz 2f # -> enabled, can't be a double fault
601	674	tm __LC_PGM_ILC+3,0x80 # check for per exception
602	675	jnz .Lpgm_svcper # -> single stepped svc
603		-1: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
	676	+2: CHECK_STACK __LC_SAVE_AREA_SYNC
604	677	aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
605		- j 4f
606		-2: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
	678	+ # CHECK_VMAP_STACK branches to stack_overflow or 5f
	679	+ CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,5f
	680	+3: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
607	681	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
608	682	lg %r15,__LC_KERNEL_STACK
609	683	lgr %r14,%r12
610	684	aghi %r14,__TASK_thread # pointer to thread_struct
611	685	lghi %r13,__LC_PGM_TDB
612	686	tm __LC_PGM_ILC+2,0x02 # check for transaction abort
613		- jz 3f
	687	+ jz 4f
614	688	mvc __THREAD_trap_tdb(256,%r14),0(%r13)
615		-3: stg %r10,__THREAD_last_break(%r14)
616		-4: lgr %r13,%r11
	689	+4: stg %r10,__THREAD_last_break(%r14)
	690	+5: lgr %r13,%r11
617	691	la %r11,STACK_FRAME_OVERHEAD(%r15)
618	692	stmg %r0,%r7,__PT_R0(%r11)
619	693	# clear user controlled registers to prevent speculative use
..	..	@@ -632,21 +706,21 @@
632	706	stg %r13,__PT_FLAGS(%r11)
633	707	stg %r10,__PT_ARGS(%r11)
634	708	tm __LC_PGM_ILC+3,0x80 # check for per exception
635		- jz 5f
	709	+ jz 6f
636	710	tmhh %r8,0x0001 # kernel per event ?
637	711	jz .Lpgm_kprobe
638	712	oi __PT_FLAGS+7(%r11),_PIF_PER_TRAP
639	713	mvc __THREAD_per_address(8,%r14),__LC_PER_ADDRESS
640	714	mvc __THREAD_per_cause(2,%r14),__LC_PER_CODE
641	715	mvc __THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID
642		-5: REENABLE_IRQS
643		- xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	716	+6: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	717	+ RESTORE_SM_CLEAR_PER
644	718	larl %r1,pgm_check_table
645	719	llgh %r10,__PT_INT_CODE+2(%r11)
646	720	nill %r10,0x007f
647		- sll %r10,2
	721	+ sll %r10,3
648	722	je .Lpgm_return
649		- lgf %r9,0(%r10,%r1) # load address of handler routine
	723	+ lg %r9,0(%r10,%r1) # load address of handler routine
650	724	lgr %r2,%r11 # pass pointer to pt_regs
651	725	BASR_EX %r14,%r9 # branch to interrupt-handler
652	726	.Lpgm_return:
..	..	@@ -661,8 +735,8 @@
661	735	# PER event in supervisor state, must be kprobes
662	736	#
663	737	.Lpgm_kprobe:
664		- REENABLE_IRQS
665	738	xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	739	+ RESTORE_SM_CLEAR_PER
666	740	lgr %r2,%r11 # pass pointer to pt_regs
667	741	brasl %r14,do_per_trap
668	742	j .Lpgm_return
..	..	@@ -672,11 +746,11 @@
672	746	#
673	747	.Lpgm_svcper:
674	748	mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
675		- lghi %r13,__TASK_thread
676	749	larl %r14,.Lsysc_per
677	750	stg %r14,__LC_RETURN_PSW+8
678	751	lghi %r14,_PIF_SYSCALL \| _PIF_PER_TRAP
679		- lpswe __LC_RETURN_PSW # branch to .Lsysc_per and enable irqs
	752	+ lpswe __LC_RETURN_PSW # branch to .Lsysc_per
	753	+ENDPROC(pgm_check_handler)
680	754
681	755	/*
682	756	* IO interrupt handler routine
..	..	@@ -687,9 +761,8 @@
687	761	BPOFF
688	762	stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
689	763	lg %r12,__LC_CURRENT
690		- larl %r13,cleanup_critical
691	764	lmg %r8,%r9,__LC_IO_OLD_PSW
692		- SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
	765	+ SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER,__LC_INT_CLOCK
693	766	stmg %r0,%r7,__PT_R0(%r11)
694	767	# clear user controlled registers to prevent speculative use
695	768	xgr %r0,%r0
..	..	@@ -705,10 +778,10 @@
705	778	stmg %r8,%r9,__PT_PSW(%r11)
706	779	mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
707	780	xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
	781	+ xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
708	782	TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ
709	783	jo .Lio_restore
710	784	TRACE_IRQS_OFF
711		- xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
712	785	.Lio_loop:
713	786	lgr %r2,%r11 # pass pointer to pt_regs
714	787	lghi %r3,IO_INTERRUPT
..	..	@@ -725,25 +798,20 @@
725	798	j .Lio_loop
726	799	.Lio_return:
727	800	LOCKDEP_SYS_EXIT
728		- TRACE_IRQS_ON
729		-.Lio_tif:
730	801	TSTMSK __TI_flags(%r12),_TIF_WORK
731	802	jnz .Lio_work # there is work to do (signals etc.)
732	803	TSTMSK __LC_CPU_FLAGS,_CIF_WORK
733	804	jnz .Lio_work
734	805	.Lio_restore:
735		- lg %r14,__LC_VDSO_PER_CPU
736		- lmg %r0,%r10,__PT_R0(%r11)
	806	+ TRACE_IRQS_ON
737	807	mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
738	808	tm __PT_PSW+1(%r11),0x01 # returning to user ?
739	809	jno .Lio_exit_kernel
740	810	BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
741		-.Lio_exit_timer:
742	811	stpt __LC_EXIT_TIMER
743		- mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
744	812	.Lio_exit_kernel:
745		- lmg %r11,%r15,__PT_R11(%r11)
746		- lpswe __LC_RETURN_PSW
	813	+ lmg %r0,%r15,__PT_R0(%r11)
	814	+ b __LC_RETURN_LPSWE
747	815	.Lio_done:
748	816
749	817	#
..	..	@@ -758,7 +826,7 @@
758	826	.Lio_work:
759	827	tm __PT_PSW+1(%r11),0x01 # returning to user ?
760	828	jo .Lio_work_user # yes -> do resched & signal
761		-#ifdef CONFIG_PREEMPT
	829	+#ifdef CONFIG_PREEMPTION
762	830	# check for preemptive scheduling
763	831	icm %r0,15,__LC_PREEMPT_COUNT
764	832	jnz .Lio_restore # preemption is disabled
..	..	@@ -771,9 +839,6 @@
771	839	xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
772	840	la %r11,STACK_FRAME_OVERHEAD(%r1)
773	841	lgr %r15,%r1
774		- # TRACE_IRQS_ON already done at .Lio_return, call
775		- # TRACE_IRQS_OFF to keep things symmetrical
776		- TRACE_IRQS_OFF
777	842	brasl %r14,preempt_schedule_irq
778	843	j .Lio_return
779	844	#else
..	..	@@ -793,17 +858,14 @@
793	858	#
794	859	# One of the work bits is on. Find out which one.
795	860	#
796		-.Lio_work_tif:
797		- TSTMSK __LC_CPU_FLAGS,_CIF_MCCK_PENDING
798		- jo .Lio_mcck_pending
799	861	TSTMSK __TI_flags(%r12),_TIF_NEED_RESCHED
800	862	jo .Lio_reschedule
801	863	#ifdef CONFIG_LIVEPATCH
802	864	TSTMSK __TI_flags(%r12),_TIF_PATCH_PENDING
803	865	jo .Lio_patch_pending
804	866	#endif
805		- TSTMSK __TI_flags(%r12),_TIF_SIGPENDING
806		- jo .Lio_sigpending
	867	+ TSTMSK __TI_flags(%r12),(_TIF_SIGPENDING\|_TIF_NOTIFY_SIGNAL)
	868	+ jnz .Lio_sigpending
807	869	TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME
808	870	jo .Lio_notify_resume
809	871	TSTMSK __TI_flags(%r12),_TIF_GUARDED_STORAGE
..	..	@@ -812,15 +874,6 @@
812	874	jo .Lio_vxrs
813	875	TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY\|_CIF_ASCE_SECONDARY)
814	876	jnz .Lio_asce
815		- j .Lio_return # beware of critical section cleanup
816		-
817		-#
818		-# _CIF_MCCK_PENDING is set, call handler
819		-#
820		-.Lio_mcck_pending:
821		- # TRACE_IRQS_ON already done at .Lio_return
822		- brasl %r14,s390_handle_mcck # TIF bit will be cleared by handler
823		- TRACE_IRQS_OFF
824	877	j .Lio_return
825	878
826	879	#
..	..	@@ -853,23 +906,19 @@
853	906	# _TIF_GUARDED_STORAGE is set, call guarded_storage_load
854	907	#
855	908	.Lio_guarded_storage:
856		- # TRACE_IRQS_ON already done at .Lio_return
857		- ssm __LC_SVC_NEW_PSW # reenable interrupts
	909	+ ENABLE_INTS_TRACE
858	910	lgr %r2,%r11 # pass pointer to pt_regs
859	911	brasl %r14,gs_load_bc_cb
860		- ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
861		- TRACE_IRQS_OFF
	912	+ DISABLE_INTS_TRACE
862	913	j .Lio_return
863	914
864	915	#
865	916	# _TIF_NEED_RESCHED is set, call schedule
866	917	#
867	918	.Lio_reschedule:
868		- # TRACE_IRQS_ON already done at .Lio_return
869		- ssm __LC_SVC_NEW_PSW # reenable interrupts
	919	+ ENABLE_INTS_TRACE
870	920	brasl %r14,schedule # call scheduler
871		- ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
872		- TRACE_IRQS_OFF
	921	+ DISABLE_INTS_TRACE
873	922	j .Lio_return
874	923
875	924	#
..	..	@@ -886,25 +935,22 @@
886	935	# _TIF_SIGPENDING or is set, call do_signal
887	936	#
888	937	.Lio_sigpending:
889		- # TRACE_IRQS_ON already done at .Lio_return
890		- ssm __LC_SVC_NEW_PSW # reenable interrupts
	938	+ ENABLE_INTS_TRACE
891	939	lgr %r2,%r11 # pass pointer to pt_regs
892	940	brasl %r14,do_signal
893		- ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
894		- TRACE_IRQS_OFF
	941	+ DISABLE_INTS_TRACE
895	942	j .Lio_return
896	943
897	944	#
898	945	# _TIF_NOTIFY_RESUME or is set, call do_notify_resume
899	946	#
900	947	.Lio_notify_resume:
901		- # TRACE_IRQS_ON already done at .Lio_return
902		- ssm __LC_SVC_NEW_PSW # reenable interrupts
	948	+ ENABLE_INTS_TRACE
903	949	lgr %r2,%r11 # pass pointer to pt_regs
904	950	brasl %r14,do_notify_resume
905		- ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
906		- TRACE_IRQS_OFF
	951	+ DISABLE_INTS_TRACE
907	952	j .Lio_return
	953	+ENDPROC(io_int_handler)
908	954
909	955	/*
910	956	* External interrupt handler routine
..	..	@@ -915,9 +961,8 @@
915	961	BPOFF
916	962	stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
917	963	lg %r12,__LC_CURRENT
918		- larl %r13,cleanup_critical
919	964	lmg %r8,%r9,__LC_EXT_OLD_PSW
920		- SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
	965	+ SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER,__LC_INT_CLOCK
921	966	stmg %r0,%r7,__PT_R0(%r11)
922	967	# clear user controlled registers to prevent speculative use
923	968	xgr %r0,%r0
..	..	@@ -936,39 +981,38 @@
936	981	mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
937	982	mvc __PT_INT_PARM_LONG(8,%r11),0(%r1)
938	983	xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
	984	+ xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
939	985	TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ
940	986	jo .Lio_restore
941	987	TRACE_IRQS_OFF
942		- xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
943	988	lgr %r2,%r11 # pass pointer to pt_regs
944	989	lghi %r3,EXT_INTERRUPT
945	990	brasl %r14,do_IRQ
946	991	j .Lio_return
	992	+ENDPROC(ext_int_handler)
947	993
948	994	/*
949		- * Load idle PSW. The second "half" of this function is in .Lcleanup_idle.
	995	+ * Load idle PSW.
950	996	*/
951	997	ENTRY(psw_idle)
952	998	stg %r14,(__SF_GPRS+8*8)(%r15)
953	999	stg %r3,__SF_EMPTY(%r15)
954		- larl %r1,.Lpsw_idle_lpsw+4
	1000	+ larl %r1,.Lpsw_idle_exit
955	1001	stg %r1,__SF_EMPTY+8(%r15)
956		-#ifdef CONFIG_SMP
957	1002	larl %r1,smp_cpu_mtid
958	1003	llgf %r1,0(%r1)
959	1004	ltgr %r1,%r1
960	1005	jz .Lpsw_idle_stcctm
961	1006	.insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+16(%r15)
962	1007	.Lpsw_idle_stcctm:
963		-#endif
964	1008	oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
965	1009	BPON
966	1010	STCK __CLOCK_IDLE_ENTER(%r2)
967	1011	stpt __TIMER_IDLE_ENTER(%r2)
968		-.Lpsw_idle_lpsw:
969	1012	lpswe __SF_EMPTY(%r15)
	1013	+.Lpsw_idle_exit:
970	1014	BR_EX %r14
971		-.Lpsw_idle_end:
	1015	+ENDPROC(psw_idle)
972	1016
973	1017	/*
974	1018	* Store floating-point controls and floating-point or vector register
..	..	@@ -978,6 +1022,7 @@
978	1022	* of the register contents at return from io or a system call.
979	1023	*/
980	1024	ENTRY(save_fpu_regs)
	1025	+ stnsm __SF_EMPTY(%r15),0xfc
981	1026	lg %r2,__LC_CURRENT
982	1027	aghi %r2,__TASK_thread
983	1028	TSTMSK __LC_CPU_FLAGS,_CIF_FPU
..	..	@@ -1009,8 +1054,10 @@
1009	1054	.Lsave_fpu_regs_done:
1010	1055	oi __LC_CPU_FLAGS+7,_CIF_FPU
1011	1056	.Lsave_fpu_regs_exit:
	1057	+ ssm __SF_EMPTY(%r15)
1012	1058	BR_EX %r14
1013	1059	.Lsave_fpu_regs_end:
	1060	+ENDPROC(save_fpu_regs)
1014	1061	EXPORT_SYMBOL(save_fpu_regs)
1015	1062
1016	1063	/*
..	..	@@ -1024,6 +1071,7 @@
1024	1071	* %r4
1025	1072	*/
1026	1073	load_fpu_regs:
	1074	+ stnsm __SF_EMPTY(%r15),0xfc
1027	1075	lg %r4,__LC_CURRENT
1028	1076	aghi %r4,__TASK_thread
1029	1077	TSTMSK __LC_CPU_FLAGS,_CIF_FPU
..	..	@@ -1055,10 +1103,10 @@
1055	1103	.Lload_fpu_regs_done:
1056	1104	ni __LC_CPU_FLAGS+7,255-_CIF_FPU
1057	1105	.Lload_fpu_regs_exit:
	1106	+ ssm __SF_EMPTY(%r15)
1058	1107	BR_EX %r14
1059	1108	.Lload_fpu_regs_end:
1060		-
1061		-.L__critical_end:
	1109	+ENDPROC(load_fpu_regs)
1062	1110
1063	1111	/*
1064	1112	* Machine check handler routines
..	..	@@ -1072,7 +1120,6 @@
1072	1120	lam %a0,%a15,__LC_AREGS_SAVE_AREA-4095(%r1) # validate acrs
1073	1121	lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs
1074	1122	lg %r12,__LC_CURRENT
1075		- larl %r13,cleanup_critical
1076	1123	lmg %r8,%r9,__LC_MCK_OLD_PSW
1077	1124	TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
1078	1125	jo .Lmcck_panic # yes -> rest of mcck code invalid
..	..	@@ -1137,7 +1184,8 @@
1137	1184	jnz 4f
1138	1185	TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
1139	1186	jno .Lmcck_panic
1140		-4: SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER
	1187	+4: ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
	1188	+ SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER,__LC_MCCK_CLOCK
1141	1189	.Lmcck_skip:
1142	1190	lghi %r14,__LC_GPREGS_SAVE_AREA+64
1143	1191	stmg %r0,%r7,__PT_R0(%r11)
..	..	@@ -1157,35 +1205,31 @@
1157	1205	xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
1158	1206	lgr %r2,%r11 # pass pointer to pt_regs
1159	1207	brasl %r14,s390_do_machine_check
1160		- tm __PT_PSW+1(%r11),0x01 # returning to user ?
1161		- jno .Lmcck_return
	1208	+ cghi %r2,0
	1209	+ je .Lmcck_return
1162	1210	lg %r1,__LC_KERNEL_STACK # switch to kernel stack
1163	1211	mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
1164	1212	xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
1165	1213	la %r11,STACK_FRAME_OVERHEAD(%r1)
1166	1214	lgr %r15,%r1
1167		- ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
1168		- TSTMSK __LC_CPU_FLAGS,_CIF_MCCK_PENDING
1169		- jno .Lmcck_return
1170	1215	TRACE_IRQS_OFF
1171	1216	brasl %r14,s390_handle_mcck
1172	1217	TRACE_IRQS_ON
1173	1218	.Lmcck_return:
1174		- lg %r14,__LC_VDSO_PER_CPU
1175	1219	lmg %r0,%r10,__PT_R0(%r11)
1176	1220	mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
1177	1221	tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
1178	1222	jno 0f
1179	1223	BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
1180	1224	stpt __LC_EXIT_TIMER
1181		- mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
1182	1225	0: lmg %r11,%r15,__PT_R11(%r11)
1183		- lpswe __LC_RETURN_MCCK_PSW
	1226	+ b __LC_RETURN_MCCK_LPSWE
1184	1227
1185	1228	.Lmcck_panic:
1186		- lg %r15,__LC_PANIC_STACK
	1229	+ lg %r15,__LC_NODAT_STACK
1187	1230	la %r11,STACK_FRAME_OVERHEAD(%r15)
1188	1231	j .Lmcck_skip
	1232	+ENDPROC(mcck_int_handler)
1189	1233
1190	1234	#
1191	1235	# PSW restart interrupt handler
..	..	@@ -1194,12 +1238,10 @@
1194	1238	ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40
1195	1239	stg %r15,__LC_SAVE_AREA_RESTART
1196	1240	lg %r15,__LC_RESTART_STACK
1197		- aghi %r15,-__PT_SIZE # create pt_regs on stack
1198		- xc 0(__PT_SIZE,%r15),0(%r15)
1199		- stmg %r0,%r14,__PT_R0(%r15)
1200		- mvc __PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
1201		- mvc __PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw
1202		- aghi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
	1241	+ xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
	1242	+ stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
	1243	+ mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
	1244	+ mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
1203	1245	xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
1204	1246	lg %r1,__LC_RESTART_FN # load fn, parm & source cpu
1205	1247	lg %r2,__LC_RESTART_DATA
..	..	@@ -1214,17 +1256,18 @@
1214	1256	2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
1215	1257	brc 2,2b
1216	1258	3: j 3b
	1259	+ENDPROC(restart_int_handler)
1217	1260
1218	1261	.section .kprobes.text, "ax"
1219	1262
1220		-#ifdef CONFIG_CHECK_STACK
	1263	+#if defined(CONFIG_CHECK_STACK) \|\| defined(CONFIG_VMAP_STACK)
1221	1264	/*
1222	1265	* The synchronous or the asynchronous stack overflowed. We are dead.
1223	1266	* No need to properly save the registers, we are going to panic anyway.
1224	1267	* Setup a pt_regs so that show_trace can provide a good call trace.
1225	1268	*/
1226		-stack_overflow:
1227		- lg %r15,__LC_PANIC_STACK # change to panic stack
	1269	+ENTRY(stack_overflow)
	1270	+ lg %r15,__LC_NODAT_STACK # change to panic stack
1228	1271	la %r11,STACK_FRAME_OVERHEAD(%r15)
1229	1272	stmg %r0,%r7,__PT_R0(%r11)
1230	1273	stmg %r8,%r9,__PT_PSW(%r11)
..	..	@@ -1233,268 +1276,29 @@
1233	1276	xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
1234	1277	lgr %r2,%r11 # pass pointer to pt_regs
1235	1278	jg kernel_stack_overflow
	1279	+ENDPROC(stack_overflow)
1236	1280	#endif
1237	1281
1238		-cleanup_critical:
1239	1282	#if IS_ENABLED(CONFIG_KVM)
1240		- clg %r9,BASED(.Lcleanup_table_sie) # .Lsie_gmap
1241		- jl 0f
1242		- clg %r9,BASED(.Lcleanup_table_sie+8)# .Lsie_done
1243		- jl .Lcleanup_sie
1244		-#endif
1245		- clg %r9,BASED(.Lcleanup_table) # system_call
1246		- jl 0f
1247		- clg %r9,BASED(.Lcleanup_table+8) # .Lsysc_do_svc
1248		- jl .Lcleanup_system_call
1249		- clg %r9,BASED(.Lcleanup_table+16) # .Lsysc_tif
1250		- jl 0f
1251		- clg %r9,BASED(.Lcleanup_table+24) # .Lsysc_restore
1252		- jl .Lcleanup_sysc_tif
1253		- clg %r9,BASED(.Lcleanup_table+32) # .Lsysc_done
1254		- jl .Lcleanup_sysc_restore
1255		- clg %r9,BASED(.Lcleanup_table+40) # .Lio_tif
1256		- jl 0f
1257		- clg %r9,BASED(.Lcleanup_table+48) # .Lio_restore
1258		- jl .Lcleanup_io_tif
1259		- clg %r9,BASED(.Lcleanup_table+56) # .Lio_done
1260		- jl .Lcleanup_io_restore
1261		- clg %r9,BASED(.Lcleanup_table+64) # psw_idle
1262		- jl 0f
1263		- clg %r9,BASED(.Lcleanup_table+72) # .Lpsw_idle_end
1264		- jl .Lcleanup_idle
1265		- clg %r9,BASED(.Lcleanup_table+80) # save_fpu_regs
1266		- jl 0f
1267		- clg %r9,BASED(.Lcleanup_table+88) # .Lsave_fpu_regs_end
1268		- jl .Lcleanup_save_fpu_regs
1269		- clg %r9,BASED(.Lcleanup_table+96) # load_fpu_regs
1270		- jl 0f
1271		- clg %r9,BASED(.Lcleanup_table+104) # .Lload_fpu_regs_end
1272		- jl .Lcleanup_load_fpu_regs
1273		-0: BR_EX %r14,%r11
1274		-
1275		- .align 8
1276		-.Lcleanup_table:
1277		- .quad system_call
1278		- .quad .Lsysc_do_svc
1279		- .quad .Lsysc_tif
1280		- .quad .Lsysc_restore
1281		- .quad .Lsysc_done
1282		- .quad .Lio_tif
1283		- .quad .Lio_restore
1284		- .quad .Lio_done
1285		- .quad psw_idle
1286		- .quad .Lpsw_idle_end
1287		- .quad save_fpu_regs
1288		- .quad .Lsave_fpu_regs_end
1289		- .quad load_fpu_regs
1290		- .quad .Lload_fpu_regs_end
1291		-
1292		-#if IS_ENABLED(CONFIG_KVM)
1293		-.Lcleanup_table_sie:
1294		- .quad .Lsie_gmap
1295		- .quad .Lsie_done
1296		-
1297	1283	.Lcleanup_sie:
1298		- cghi %r11,__LC_SAVE_AREA_ASYNC #Is this in normal interrupt?
1299		- je 1f
1300		- slg %r9,BASED(.Lsie_crit_mcck_start)
1301		- clg %r9,BASED(.Lsie_crit_mcck_length)
1302		- jh 1f
1303		- oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
1304		-1: BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP\|_TIF_ISOLATE_BP_GUEST)
	1284	+ cghi %r11,__LC_SAVE_AREA_ASYNC #Is this in normal interrupt?
	1285	+ je 1f
	1286	+ larl %r13,.Lsie_entry
	1287	+ slgr %r9,%r13
	1288	+ lghi %r13,.Lsie_skip - .Lsie_entry
	1289	+ clgr %r9,%r13
	1290	+ jh 1f
	1291	+ oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
	1292	+1: BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP\|_TIF_ISOLATE_BP_GUEST)
1305	1293	lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer
1306	1294	ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
1307	1295	lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
1308	1296	larl %r9,sie_exit # skip forward to sie_exit
1309	1297	BR_EX %r14,%r11
1310		-#endif
1311	1298
1312		-.Lcleanup_system_call:
1313		- # check if stpt has been executed
1314		- clg %r9,BASED(.Lcleanup_system_call_insn)
1315		- jh 0f
1316		- mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
1317		- cghi %r11,__LC_SAVE_AREA_ASYNC
1318		- je 0f
1319		- mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
1320		-0: # check if stmg has been executed
1321		- clg %r9,BASED(.Lcleanup_system_call_insn+8)
1322		- jh 0f
1323		- mvc __LC_SAVE_AREA_SYNC(64),0(%r11)
1324		-0: # check if base register setup + TIF bit load has been done
1325		- clg %r9,BASED(.Lcleanup_system_call_insn+16)
1326		- jhe 0f
1327		- # set up saved register r12 task struct pointer
1328		- stg %r12,32(%r11)
1329		- # set up saved register r13 __TASK_thread offset
1330		- mvc 40(8,%r11),BASED(.Lcleanup_system_call_const)
1331		-0: # check if the user time update has been done
1332		- clg %r9,BASED(.Lcleanup_system_call_insn+24)
1333		- jh 0f
1334		- lg %r15,__LC_EXIT_TIMER
1335		- slg %r15,__LC_SYNC_ENTER_TIMER
1336		- alg %r15,__LC_USER_TIMER
1337		- stg %r15,__LC_USER_TIMER
1338		-0: # check if the system time update has been done
1339		- clg %r9,BASED(.Lcleanup_system_call_insn+32)
1340		- jh 0f
1341		- lg %r15,__LC_LAST_UPDATE_TIMER
1342		- slg %r15,__LC_EXIT_TIMER
1343		- alg %r15,__LC_SYSTEM_TIMER
1344		- stg %r15,__LC_SYSTEM_TIMER
1345		-0: # update accounting time stamp
1346		- mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
1347		- BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
1348		- # set up saved register r11
1349		- lg %r15,__LC_KERNEL_STACK
1350		- la %r9,STACK_FRAME_OVERHEAD(%r15)
1351		- stg %r9,24(%r11) # r11 pt_regs pointer
1352		- # fill pt_regs
1353		- mvc __PT_R8(64,%r9),__LC_SAVE_AREA_SYNC
1354		- stmg %r0,%r7,__PT_R0(%r9)
1355		- mvc __PT_PSW(16,%r9),__LC_SVC_OLD_PSW
1356		- mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC
1357		- xc __PT_FLAGS(8,%r9),__PT_FLAGS(%r9)
1358		- mvi __PT_FLAGS+7(%r9),_PIF_SYSCALL
1359		- # setup saved register r15
1360		- stg %r15,56(%r11) # r15 stack pointer
1361		- # set new psw address and exit
1362		- larl %r9,.Lsysc_do_svc
1363		- BR_EX %r14,%r11
1364		-.Lcleanup_system_call_insn:
1365		- .quad system_call
1366		- .quad .Lsysc_stmg
1367		- .quad .Lsysc_per
1368		- .quad .Lsysc_vtime+36
1369		- .quad .Lsysc_vtime+42
1370		-.Lcleanup_system_call_const:
1371		- .quad __TASK_thread
1372		-
1373		-.Lcleanup_sysc_tif:
1374		- larl %r9,.Lsysc_tif
1375		- BR_EX %r14,%r11
1376		-
1377		-.Lcleanup_sysc_restore:
1378		- # check if stpt has been executed
1379		- clg %r9,BASED(.Lcleanup_sysc_restore_insn)
1380		- jh 0f
1381		- mvc __LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
1382		- cghi %r11,__LC_SAVE_AREA_ASYNC
1383		- je 0f
1384		- mvc __LC_EXIT_TIMER(8),__LC_MCCK_ENTER_TIMER
1385		-0: clg %r9,BASED(.Lcleanup_sysc_restore_insn+8)
1386		- je 1f
1387		- lg %r9,24(%r11) # get saved pointer to pt_regs
1388		- mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
1389		- mvc 0(64,%r11),__PT_R8(%r9)
1390		- lmg %r0,%r7,__PT_R0(%r9)
1391		-1: lmg %r8,%r9,__LC_RETURN_PSW
1392		- BR_EX %r14,%r11
1393		-.Lcleanup_sysc_restore_insn:
1394		- .quad .Lsysc_exit_timer
1395		- .quad .Lsysc_done - 4
1396		-
1397		-.Lcleanup_io_tif:
1398		- larl %r9,.Lio_tif
1399		- BR_EX %r14,%r11
1400		-
1401		-.Lcleanup_io_restore:
1402		- # check if stpt has been executed
1403		- clg %r9,BASED(.Lcleanup_io_restore_insn)
1404		- jh 0f
1405		- mvc __LC_EXIT_TIMER(8),__LC_MCCK_ENTER_TIMER
1406		-0: clg %r9,BASED(.Lcleanup_io_restore_insn+8)
1407		- je 1f
1408		- lg %r9,24(%r11) # get saved r11 pointer to pt_regs
1409		- mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
1410		- mvc 0(64,%r11),__PT_R8(%r9)
1411		- lmg %r0,%r7,__PT_R0(%r9)
1412		-1: lmg %r8,%r9,__LC_RETURN_PSW
1413		- BR_EX %r14,%r11
1414		-.Lcleanup_io_restore_insn:
1415		- .quad .Lio_exit_timer
1416		- .quad .Lio_done - 4
1417		-
1418		-.Lcleanup_idle:
1419		- ni __LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT
1420		- # copy interrupt clock & cpu timer
1421		- mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
1422		- mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
1423		- cghi %r11,__LC_SAVE_AREA_ASYNC
1424		- je 0f
1425		- mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
1426		- mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
1427		-0: # check if stck & stpt have been executed
1428		- clg %r9,BASED(.Lcleanup_idle_insn)
1429		- jhe 1f
1430		- mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
1431		- mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r2)
1432		-1: # calculate idle cycles
1433		-#ifdef CONFIG_SMP
1434		- clg %r9,BASED(.Lcleanup_idle_insn)
1435		- jl 3f
1436		- larl %r1,smp_cpu_mtid
1437		- llgf %r1,0(%r1)
1438		- ltgr %r1,%r1
1439		- jz 3f
1440		- .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15)
1441		- larl %r3,mt_cycles
1442		- ag %r3,__LC_PERCPU_OFFSET
1443		- la %r4,__SF_EMPTY+16(%r15)
1444		-2: lg %r0,0(%r3)
1445		- slg %r0,0(%r4)
1446		- alg %r0,64(%r4)
1447		- stg %r0,0(%r3)
1448		- la %r3,8(%r3)
1449		- la %r4,8(%r4)
1450		- brct %r1,2b
1451		-#endif
1452		-3: # account system time going idle
1453		- lg %r9,__LC_STEAL_TIMER
1454		- alg %r9,__CLOCK_IDLE_ENTER(%r2)
1455		- slg %r9,__LC_LAST_UPDATE_CLOCK
1456		- stg %r9,__LC_STEAL_TIMER
1457		- mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
1458		- lg %r9,__LC_SYSTEM_TIMER
1459		- alg %r9,__LC_LAST_UPDATE_TIMER
1460		- slg %r9,__TIMER_IDLE_ENTER(%r2)
1461		- stg %r9,__LC_SYSTEM_TIMER
1462		- mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
1463		- # prepare return psw
1464		- nihh %r8,0xfcfd # clear irq & wait state bits
1465		- lg %r9,48(%r11) # return from psw_idle
1466		- BR_EX %r14,%r11
1467		-.Lcleanup_idle_insn:
1468		- .quad .Lpsw_idle_lpsw
1469		-
1470		-.Lcleanup_save_fpu_regs:
1471		- larl %r9,save_fpu_regs
1472		- BR_EX %r14,%r11
1473		-
1474		-.Lcleanup_load_fpu_regs:
1475		- larl %r9,load_fpu_regs
1476		- BR_EX %r14,%r11
1477		-
1478		-/*
1479		- * Integer constants
1480		- */
1481		- .align 8
1482		-.Lcritical_start:
1483		- .quad .L__critical_start
1484		-.Lcritical_length:
1485		- .quad .L__critical_end - .L__critical_start
1486		-#if IS_ENABLED(CONFIG_KVM)
1487		-.Lsie_critical_start:
1488		- .quad .Lsie_gmap
1489		-.Lsie_critical_length:
1490		- .quad .Lsie_done - .Lsie_gmap
1491		-.Lsie_crit_mcck_start:
1492		- .quad .Lsie_entry
1493		-.Lsie_crit_mcck_length:
1494		- .quad .Lsie_skip - .Lsie_entry
1495	1299	#endif
1496	1300	.section .rodata, "a"
1497		-#define SYSCALL(esame,emu) .long esame
	1301	+#define SYSCALL(esame,emu) .quad __s390x_ ## esame
1498	1302	.globl sys_call_table
1499	1303	sys_call_table:
1500	1304	#include "asm/syscall_table.h"
..	..	@@ -1502,7 +1306,7 @@
1502	1306
1503	1307	#ifdef CONFIG_COMPAT
1504	1308
1505		-#define SYSCALL(esame,emu) .long emu
	1309	+#define SYSCALL(esame,emu) .quad __s390_ ## emu
1506	1310	.globl sys_call_table_emu
1507	1311	sys_call_table_emu:
1508	1312	#include "asm/syscall_table.h"