~hc/RK356X_SDK_RELEASE.git

..	..	@@ -2,7 +2,7 @@
2	2	/*
3	3	* hosting IBM Z kernel virtual machines (s390x)
4	4	*
5		- * Copyright IBM Corp. 2008, 2018
	5	+ * Copyright IBM Corp. 2008, 2020
6	6	*
7	7	* Author(s): Carsten Otte <cotte@de.ibm.com>
8	8	* Christian Borntraeger <borntraeger@de.ibm.com>
..	..	@@ -10,6 +10,9 @@
10	10	* Christian Ehrhardt <ehrhardt@de.ibm.com>
11	11	* Jason J. Herne <jjherne@us.ibm.com>
12	12	*/
	13	+
	14	+#define KMSG_COMPONENT "kvm-s390"
	15	+#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
13	16
14	17	#include <linux/compiler.h>
15	18	#include <linux/err.h>
..	..	@@ -28,11 +31,11 @@
28	31	#include <linux/bitmap.h>
29	32	#include <linux/sched/signal.h>
30	33	#include <linux/string.h>
	34	+#include <linux/pgtable.h>
31	35
32	36	#include <asm/asm-offsets.h>
33	37	#include <asm/lowcore.h>
34	38	#include <asm/stp.h>
35		-#include <asm/pgtable.h>
36	39	#include <asm/gmap.h>
37	40	#include <asm/nmi.h>
38	41	#include <asm/switch_to.h>
..	..	@@ -40,12 +43,10 @@
40	43	#include <asm/sclp.h>
41	44	#include <asm/cpacf.h>
42	45	#include <asm/timex.h>
	46	+#include <asm/ap.h>
	47	+#include <asm/uv.h>
43	48	#include "kvm-s390.h"
44	49	#include "gaccess.h"
45		-
46		-#define KMSG_COMPONENT "kvm-s390"
47		-#undef pr_fmt
48		-#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
49	50
50	51	#define CREATE_TRACE_POINTS
51	52	#include "trace.h"
..	..	@@ -56,108 +57,109 @@
56	57	#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
57	58	(KVM_MAX_VCPUS + LOCAL_IRQS))
58	59
59		-#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
60		-#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
61		-
62	60	struct kvm_stats_debugfs_item debugfs_entries[] = {
63		- { "userspace_handled", VCPU_STAT(exit_userspace) },
64		- { "exit_null", VCPU_STAT(exit_null) },
65		- { "exit_validity", VCPU_STAT(exit_validity) },
66		- { "exit_stop_request", VCPU_STAT(exit_stop_request) },
67		- { "exit_external_request", VCPU_STAT(exit_external_request) },
68		- { "exit_io_request", VCPU_STAT(exit_io_request) },
69		- { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
70		- { "exit_instruction", VCPU_STAT(exit_instruction) },
71		- { "exit_pei", VCPU_STAT(exit_pei) },
72		- { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
73		- { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
74		- { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
75		- { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
76		- { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
77		- { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
78		- { "halt_wakeup", VCPU_STAT(halt_wakeup) },
79		- { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
80		- { "instruction_lctl", VCPU_STAT(instruction_lctl) },
81		- { "instruction_stctl", VCPU_STAT(instruction_stctl) },
82		- { "instruction_stctg", VCPU_STAT(instruction_stctg) },
83		- { "deliver_ckc", VCPU_STAT(deliver_ckc) },
84		- { "deliver_cputm", VCPU_STAT(deliver_cputm) },
85		- { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
86		- { "deliver_external_call", VCPU_STAT(deliver_external_call) },
87		- { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
88		- { "deliver_virtio", VCPU_STAT(deliver_virtio) },
89		- { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
90		- { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
91		- { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
92		- { "deliver_program", VCPU_STAT(deliver_program) },
93		- { "deliver_io", VCPU_STAT(deliver_io) },
94		- { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
95		- { "exit_wait_state", VCPU_STAT(exit_wait_state) },
96		- { "inject_ckc", VCPU_STAT(inject_ckc) },
97		- { "inject_cputm", VCPU_STAT(inject_cputm) },
98		- { "inject_external_call", VCPU_STAT(inject_external_call) },
99		- { "inject_float_mchk", VM_STAT(inject_float_mchk) },
100		- { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
101		- { "inject_io", VM_STAT(inject_io) },
102		- { "inject_mchk", VCPU_STAT(inject_mchk) },
103		- { "inject_pfault_done", VM_STAT(inject_pfault_done) },
104		- { "inject_program", VCPU_STAT(inject_program) },
105		- { "inject_restart", VCPU_STAT(inject_restart) },
106		- { "inject_service_signal", VM_STAT(inject_service_signal) },
107		- { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
108		- { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
109		- { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
110		- { "inject_virtio", VM_STAT(inject_virtio) },
111		- { "instruction_epsw", VCPU_STAT(instruction_epsw) },
112		- { "instruction_gs", VCPU_STAT(instruction_gs) },
113		- { "instruction_io_other", VCPU_STAT(instruction_io_other) },
114		- { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
115		- { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
116		- { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
117		- { "instruction_ptff", VCPU_STAT(instruction_ptff) },
118		- { "instruction_stidp", VCPU_STAT(instruction_stidp) },
119		- { "instruction_sck", VCPU_STAT(instruction_sck) },
120		- { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
121		- { "instruction_spx", VCPU_STAT(instruction_spx) },
122		- { "instruction_stpx", VCPU_STAT(instruction_stpx) },
123		- { "instruction_stap", VCPU_STAT(instruction_stap) },
124		- { "instruction_iske", VCPU_STAT(instruction_iske) },
125		- { "instruction_ri", VCPU_STAT(instruction_ri) },
126		- { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
127		- { "instruction_sske", VCPU_STAT(instruction_sske) },
128		- { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
129		- { "instruction_essa", VCPU_STAT(instruction_essa) },
130		- { "instruction_stsi", VCPU_STAT(instruction_stsi) },
131		- { "instruction_stfl", VCPU_STAT(instruction_stfl) },
132		- { "instruction_tb", VCPU_STAT(instruction_tb) },
133		- { "instruction_tpi", VCPU_STAT(instruction_tpi) },
134		- { "instruction_tprot", VCPU_STAT(instruction_tprot) },
135		- { "instruction_tsch", VCPU_STAT(instruction_tsch) },
136		- { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
137		- { "instruction_sie", VCPU_STAT(instruction_sie) },
138		- { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
139		- { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
140		- { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
141		- { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
142		- { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
143		- { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
144		- { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
145		- { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
146		- { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
147		- { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
148		- { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
149		- { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
150		- { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
151		- { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
152		- { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
153		- { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
154		- { "instruction_diag_10", VCPU_STAT(diagnose_10) },
155		- { "instruction_diag_44", VCPU_STAT(diagnose_44) },
156		- { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
157		- { "instruction_diag_258", VCPU_STAT(diagnose_258) },
158		- { "instruction_diag_308", VCPU_STAT(diagnose_308) },
159		- { "instruction_diag_500", VCPU_STAT(diagnose_500) },
160		- { "instruction_diag_other", VCPU_STAT(diagnose_other) },
	61	+ VCPU_STAT("userspace_handled", exit_userspace),
	62	+ VCPU_STAT("exit_null", exit_null),
	63	+ VCPU_STAT("exit_validity", exit_validity),
	64	+ VCPU_STAT("exit_stop_request", exit_stop_request),
	65	+ VCPU_STAT("exit_external_request", exit_external_request),
	66	+ VCPU_STAT("exit_io_request", exit_io_request),
	67	+ VCPU_STAT("exit_external_interrupt", exit_external_interrupt),
	68	+ VCPU_STAT("exit_instruction", exit_instruction),
	69	+ VCPU_STAT("exit_pei", exit_pei),
	70	+ VCPU_STAT("exit_program_interruption", exit_program_interruption),
	71	+ VCPU_STAT("exit_instr_and_program_int", exit_instr_and_program),
	72	+ VCPU_STAT("exit_operation_exception", exit_operation_exception),
	73	+ VCPU_STAT("halt_successful_poll", halt_successful_poll),
	74	+ VCPU_STAT("halt_attempted_poll", halt_attempted_poll),
	75	+ VCPU_STAT("halt_poll_invalid", halt_poll_invalid),
	76	+ VCPU_STAT("halt_no_poll_steal", halt_no_poll_steal),
	77	+ VCPU_STAT("halt_wakeup", halt_wakeup),
	78	+ VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns),
	79	+ VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns),
	80	+ VCPU_STAT("instruction_lctlg", instruction_lctlg),
	81	+ VCPU_STAT("instruction_lctl", instruction_lctl),
	82	+ VCPU_STAT("instruction_stctl", instruction_stctl),
	83	+ VCPU_STAT("instruction_stctg", instruction_stctg),
	84	+ VCPU_STAT("deliver_ckc", deliver_ckc),
	85	+ VCPU_STAT("deliver_cputm", deliver_cputm),
	86	+ VCPU_STAT("deliver_emergency_signal", deliver_emergency_signal),
	87	+ VCPU_STAT("deliver_external_call", deliver_external_call),
	88	+ VCPU_STAT("deliver_service_signal", deliver_service_signal),
	89	+ VCPU_STAT("deliver_virtio", deliver_virtio),
	90	+ VCPU_STAT("deliver_stop_signal", deliver_stop_signal),
	91	+ VCPU_STAT("deliver_prefix_signal", deliver_prefix_signal),
	92	+ VCPU_STAT("deliver_restart_signal", deliver_restart_signal),
	93	+ VCPU_STAT("deliver_program", deliver_program),
	94	+ VCPU_STAT("deliver_io", deliver_io),
	95	+ VCPU_STAT("deliver_machine_check", deliver_machine_check),
	96	+ VCPU_STAT("exit_wait_state", exit_wait_state),
	97	+ VCPU_STAT("inject_ckc", inject_ckc),
	98	+ VCPU_STAT("inject_cputm", inject_cputm),
	99	+ VCPU_STAT("inject_external_call", inject_external_call),
	100	+ VM_STAT("inject_float_mchk", inject_float_mchk),
	101	+ VCPU_STAT("inject_emergency_signal", inject_emergency_signal),
	102	+ VM_STAT("inject_io", inject_io),
	103	+ VCPU_STAT("inject_mchk", inject_mchk),
	104	+ VM_STAT("inject_pfault_done", inject_pfault_done),
	105	+ VCPU_STAT("inject_program", inject_program),
	106	+ VCPU_STAT("inject_restart", inject_restart),
	107	+ VM_STAT("inject_service_signal", inject_service_signal),
	108	+ VCPU_STAT("inject_set_prefix", inject_set_prefix),
	109	+ VCPU_STAT("inject_stop_signal", inject_stop_signal),
	110	+ VCPU_STAT("inject_pfault_init", inject_pfault_init),
	111	+ VM_STAT("inject_virtio", inject_virtio),
	112	+ VCPU_STAT("instruction_epsw", instruction_epsw),
	113	+ VCPU_STAT("instruction_gs", instruction_gs),
	114	+ VCPU_STAT("instruction_io_other", instruction_io_other),
	115	+ VCPU_STAT("instruction_lpsw", instruction_lpsw),
	116	+ VCPU_STAT("instruction_lpswe", instruction_lpswe),
	117	+ VCPU_STAT("instruction_pfmf", instruction_pfmf),
	118	+ VCPU_STAT("instruction_ptff", instruction_ptff),
	119	+ VCPU_STAT("instruction_stidp", instruction_stidp),
	120	+ VCPU_STAT("instruction_sck", instruction_sck),
	121	+ VCPU_STAT("instruction_sckpf", instruction_sckpf),
	122	+ VCPU_STAT("instruction_spx", instruction_spx),
	123	+ VCPU_STAT("instruction_stpx", instruction_stpx),
	124	+ VCPU_STAT("instruction_stap", instruction_stap),
	125	+ VCPU_STAT("instruction_iske", instruction_iske),
	126	+ VCPU_STAT("instruction_ri", instruction_ri),
	127	+ VCPU_STAT("instruction_rrbe", instruction_rrbe),
	128	+ VCPU_STAT("instruction_sske", instruction_sske),
	129	+ VCPU_STAT("instruction_ipte_interlock", instruction_ipte_interlock),
	130	+ VCPU_STAT("instruction_essa", instruction_essa),
	131	+ VCPU_STAT("instruction_stsi", instruction_stsi),
	132	+ VCPU_STAT("instruction_stfl", instruction_stfl),
	133	+ VCPU_STAT("instruction_tb", instruction_tb),
	134	+ VCPU_STAT("instruction_tpi", instruction_tpi),
	135	+ VCPU_STAT("instruction_tprot", instruction_tprot),
	136	+ VCPU_STAT("instruction_tsch", instruction_tsch),
	137	+ VCPU_STAT("instruction_sthyi", instruction_sthyi),
	138	+ VCPU_STAT("instruction_sie", instruction_sie),
	139	+ VCPU_STAT("instruction_sigp_sense", instruction_sigp_sense),
	140	+ VCPU_STAT("instruction_sigp_sense_running", instruction_sigp_sense_running),
	141	+ VCPU_STAT("instruction_sigp_external_call", instruction_sigp_external_call),
	142	+ VCPU_STAT("instruction_sigp_emergency", instruction_sigp_emergency),
	143	+ VCPU_STAT("instruction_sigp_cond_emergency", instruction_sigp_cond_emergency),
	144	+ VCPU_STAT("instruction_sigp_start", instruction_sigp_start),
	145	+ VCPU_STAT("instruction_sigp_stop", instruction_sigp_stop),
	146	+ VCPU_STAT("instruction_sigp_stop_store_status", instruction_sigp_stop_store_status),
	147	+ VCPU_STAT("instruction_sigp_store_status", instruction_sigp_store_status),
	148	+ VCPU_STAT("instruction_sigp_store_adtl_status", instruction_sigp_store_adtl_status),
	149	+ VCPU_STAT("instruction_sigp_set_arch", instruction_sigp_arch),
	150	+ VCPU_STAT("instruction_sigp_set_prefix", instruction_sigp_prefix),
	151	+ VCPU_STAT("instruction_sigp_restart", instruction_sigp_restart),
	152	+ VCPU_STAT("instruction_sigp_cpu_reset", instruction_sigp_cpu_reset),
	153	+ VCPU_STAT("instruction_sigp_init_cpu_reset", instruction_sigp_init_cpu_reset),
	154	+ VCPU_STAT("instruction_sigp_unknown", instruction_sigp_unknown),
	155	+ VCPU_STAT("instruction_diag_10", diagnose_10),
	156	+ VCPU_STAT("instruction_diag_44", diagnose_44),
	157	+ VCPU_STAT("instruction_diag_9c", diagnose_9c),
	158	+ VCPU_STAT("diag_9c_ignored", diagnose_9c_ignored),
	159	+ VCPU_STAT("instruction_diag_258", diagnose_258),
	160	+ VCPU_STAT("instruction_diag_308", diagnose_308),
	161	+ VCPU_STAT("instruction_diag_500", diagnose_500),
	162	+ VCPU_STAT("instruction_diag_other", diagnose_other),
161	163	{ NULL }
162	164	};
163	165
..	..	@@ -176,6 +178,16 @@
176	178	static int hpage;
177	179	module_param(hpage, int, 0444);
178	180	MODULE_PARM_DESC(hpage, "1m huge page backing support");
	181	+
	182	+/* maximum percentage of steal time for polling. >100 is treated like 100 */
	183	+static u8 halt_poll_max_steal = 10;
	184	+module_param(halt_poll_max_steal, byte, 0644);
	185	+MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
	186	+
	187	+/* if set to true, the GISA will be initialized and used if available */
	188	+static bool use_gisa = true;
	189	+module_param(use_gisa, bool, 0644);
	190	+MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
179	191
180	192	/*
181	193	* For now we handle at most 16 double words as this is what the s390 base
..	..	@@ -213,6 +225,7 @@
213	225	static struct gmap_notifier gmap_notifier;
214	226	static struct gmap_notifier vsie_gmap_notifier;
215	227	debug_info_t *kvm_s390_dbf;
	228	+debug_info_t *kvm_s390_dbf_uv;
216	229
217	230	/* Section: not file related */
218	231	int kvm_arch_hardware_enable(void)
..	..	@@ -221,8 +234,15 @@
221	234	return 0;
222	235	}
223	236
	237	+int kvm_arch_check_processor_compat(void *opaque)
	238	+{
	239	+ return 0;
	240	+}
	241	+
	242	+/* forward declarations */
224	243	static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
225	244	unsigned long end);
	245	+static int sca_switch_to_extended(struct kvm *kvm);
226	246
227	247	static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
228	248	{
..	..	@@ -281,7 +301,7 @@
281	301	.notifier_call = kvm_clock_sync,
282	302	};
283	303
284		-int kvm_arch_hardware_setup(void)
	304	+int kvm_arch_hardware_setup(void *opaque)
285	305	{
286	306	gmap_notifier.notifier_call = kvm_gmap_notifier;
287	307	gmap_register_pte_notifier(&gmap_notifier);
..	..	@@ -307,19 +327,35 @@
307	327
308	328	static inline int plo_test_bit(unsigned char nr)
309	329	{
310		- register unsigned long r0 asm("0") = (unsigned long) nr \| 0x100;
	330	+ unsigned long function = (unsigned long)nr \| 0x100;
311	331	int cc;
312	332
313	333	asm volatile(
	334	+ " lgr 0,%[function]\n"
314	335	/* Parameter registers are ignored for "test bit" */
315	336	" plo 0,0,0,0(0)\n"
316	337	" ipm %0\n"
317	338	" srl %0,28\n"
318	339	: "=d" (cc)
319		- : "d" (r0)
320		- : "cc");
	340	+ : [function] "d" (function)
	341	+ : "cc", "0");
321	342	return cc == 0;
322	343	}
	344	+
	345	+static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
	346	+{
	347	+ asm volatile(
	348	+ " lghi 0,0\n"
	349	+ " lgr 1,%[query]\n"
	350	+ /* Parameter registers are ignored */
	351	+ " .insn rrf,%[opc] << 16,2,4,6,0\n"
	352	+ :
	353	+ : [query] "d" ((unsigned long)query), [opc] "i" (opcode)
	354	+ : "cc", "memory", "0", "1");
	355	+}
	356	+
	357	+#define INSN_SORTL 0xb938
	358	+#define INSN_DFLTCC 0xb939
323	359
324	360	static void kvm_s390_cpu_feat_init(void)
325	361	{
..	..	@@ -367,6 +403,16 @@
367	403	if (test_facility(146)) /* MSA8 */
368	404	__cpacf_query(CPACF_KMA, (cpacf_mask_t *)
369	405	kvm_s390_available_subfunc.kma);
	406	+
	407	+ if (test_facility(155)) /* MSA9 */
	408	+ __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
	409	+ kvm_s390_available_subfunc.kdsa);
	410	+
	411	+ if (test_facility(150)) /* SORTL */
	412	+ __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
	413	+
	414	+ if (test_facility(151)) /* DFLTCC */
	415	+ __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
370	416
371	417	if (MACHINE_HAS_ESOP)
372	418	allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
..	..	@@ -416,35 +462,45 @@
416	462
417	463	int kvm_arch_init(void *opaque)
418	464	{
419		- int rc;
	465	+ int rc = -ENOMEM;
420	466
421	467	kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
422	468	if (!kvm_s390_dbf)
423	469	return -ENOMEM;
424	470
425		- if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
426		- rc = -ENOMEM;
427		- goto out_debug_unreg;
428		- }
	471	+ kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
	472	+ if (!kvm_s390_dbf_uv)
	473	+ goto out;
	474	+
	475	+ if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) \|\|
	476	+ debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
	477	+ goto out;
429	478
430	479	kvm_s390_cpu_feat_init();
431	480
432	481	/* Register floating interrupt controller interface. */
433	482	rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
434	483	if (rc) {
435		- pr_err("Failed to register FLIC rc=%d\n", rc);
436		- goto out_debug_unreg;
	484	+ pr_err("A FLIC registration call failed with rc=%d\n", rc);
	485	+ goto out;
437	486	}
	487	+
	488	+ rc = kvm_s390_gib_init(GAL_ISC);
	489	+ if (rc)
	490	+ goto out;
	491	+
438	492	return 0;
439	493
440		-out_debug_unreg:
441		- debug_unregister(kvm_s390_dbf);
	494	+out:
	495	+ kvm_arch_exit();
442	496	return rc;
443	497	}
444	498
445	499	void kvm_arch_exit(void)
446	500	{
	501	+ kvm_s390_gib_destroy();
447	502	debug_unregister(kvm_s390_dbf);
	503	+ debug_unregister(kvm_s390_dbf_uv);
448	504	}
449	505
450	506	/* Section: device related */
..	..	@@ -474,7 +530,6 @@
474	530	case KVM_CAP_S390_CSS_SUPPORT:
475	531	case KVM_CAP_IOEVENTFD:
476	532	case KVM_CAP_DEVICE_CTRL:
477		- case KVM_CAP_ENABLE_CAP_VM:
478	533	case KVM_CAP_S390_IRQCHIP:
479	534	case KVM_CAP_VM_ATTRIBUTES:
480	535	case KVM_CAP_MP_STATE:
..	..	@@ -488,6 +543,9 @@
488	543	case KVM_CAP_S390_CMMA_MIGRATION:
489	544	case KVM_CAP_S390_AIS:
490	545	case KVM_CAP_S390_AIS_MIGRATION:
	546	+ case KVM_CAP_S390_VCPU_RESETS:
	547	+ case KVM_CAP_SET_GUEST_DEBUG:
	548	+ case KVM_CAP_S390_DIAG318:
491	549	r = 1;
492	550	break;
493	551	case KVM_CAP_S390_HPAGE_1M:
..	..	@@ -507,9 +565,6 @@
507	565	else if (sclp.has_esca && sclp.has_64bscao)
508	566	r = KVM_S390_ESCA_CPU_SLOTS;
509	567	break;
510		- case KVM_CAP_NR_MEMSLOTS:
511		- r = KVM_USER_MEM_SLOTS;
512		- break;
513	568	case KVM_CAP_S390_COW:
514	569	r = MACHINE_HAS_ESOP;
515	570	break;
..	..	@@ -525,14 +580,16 @@
525	580	case KVM_CAP_S390_BPB:
526	581	r = test_facility(82);
527	582	break;
	583	+ case KVM_CAP_S390_PROTECTED:
	584	+ r = is_prot_virt_host();
	585	+ break;
528	586	default:
529	587	r = 0;
530	588	}
531	589	return r;
532	590	}
533	591
534		-static void kvm_s390_sync_dirty_log(struct kvm *kvm,
535		- struct kvm_memory_slot *memslot)
	592	+void kvm_arch_sync_dirty_log(struct kvm kvm, struct kvm_memory_slot memslot)
536	593	{
537	594	int i;
538	595	gfn_t cur_gfn, last_gfn;
..	..	@@ -573,9 +630,8 @@
573	630	{
574	631	int r;
575	632	unsigned long n;
576		- struct kvm_memslots *slots;
577	633	struct kvm_memory_slot *memslot;
578		- int is_dirty = 0;
	634	+ int is_dirty;
579	635
580	636	if (kvm_is_ucontrol(kvm))
581	637	return -EINVAL;
..	..	@@ -586,14 +642,7 @@
586	642	if (log->slot >= KVM_USER_MEM_SLOTS)
587	643	goto out;
588	644
589		- slots = kvm_memslots(kvm);
590		- memslot = id_to_memslot(slots, log->slot);
591		- r = -ENOENT;
592		- if (!memslot->dirty_bitmap)
593		- goto out;
594		-
595		- kvm_s390_sync_dirty_log(kvm, memslot);
596		- r = kvm_get_dirty_log(kvm, log, &is_dirty);
	645	+ r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
597	646	if (r)
598	647	goto out;
599	648
..	..	@@ -618,7 +667,7 @@
618	667	}
619	668	}
620	669
621		-static int kvm_vm_ioctl_enable_cap(struct kvm kvm, struct kvm_enable_cap cap)
	670	+int kvm_vm_ioctl_enable_cap(struct kvm kvm, struct kvm_enable_cap cap)
622	671	{
623	672	int r;
624	673
..	..	@@ -650,6 +699,14 @@
650	699	if (test_facility(135)) {
651	700	set_kvm_facility(kvm->arch.model.fac_mask, 135);
652	701	set_kvm_facility(kvm->arch.model.fac_list, 135);
	702	+ }
	703	+ if (test_facility(148)) {
	704	+ set_kvm_facility(kvm->arch.model.fac_mask, 148);
	705	+ set_kvm_facility(kvm->arch.model.fac_list, 148);
	706	+ }
	707	+ if (test_facility(152)) {
	708	+ set_kvm_facility(kvm->arch.model.fac_mask, 152);
	709	+ set_kvm_facility(kvm->arch.model.fac_list, 152);
653	710	}
654	711	r = 0;
655	712	} else
..	..	@@ -707,9 +764,9 @@
707	764	r = -EINVAL;
708	765	else {
709	766	r = 0;
710		- down_write(&kvm->mm->mmap_sem);
	767	+ mmap_write_lock(kvm->mm);
711	768	kvm->mm->context.allow_gmap_hpage_1m = 1;
712		- up_write(&kvm->mm->mmap_sem);
	769	+ mmap_write_unlock(kvm->mm);
713	770	/*
714	771	* We might have to create fake 4k page
715	772	* tables. To avoid that the hardware works on
..	..	@@ -856,20 +913,24 @@
856	913
857	914	kvm_s390_vcpu_block_all(kvm);
858	915
859		- kvm_for_each_vcpu(i, vcpu, kvm)
	916	+ kvm_for_each_vcpu(i, vcpu, kvm) {
860	917	kvm_s390_vcpu_crypto_setup(vcpu);
	918	+ /* recreate the shadow crycb by leaving the VSIE handler */
	919	+ kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
	920	+ }
861	921
862	922	kvm_s390_vcpu_unblock_all(kvm);
863	923	}
864	924
865	925	static int kvm_s390_vm_set_crypto(struct kvm kvm, struct kvm_device_attr attr)
866	926	{
867		- if (!test_kvm_facility(kvm, 76))
868		- return -EINVAL;
869		-
870	927	mutex_lock(&kvm->lock);
871	928	switch (attr->attr) {
872	929	case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
	930	+ if (!test_kvm_facility(kvm, 76)) {
	931	+ mutex_unlock(&kvm->lock);
	932	+ return -EINVAL;
	933	+ }
873	934	get_random_bytes(
874	935	kvm->arch.crypto.crycb->aes_wrapping_key_mask,
875	936	sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
..	..	@@ -877,6 +938,10 @@
877	938	VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
878	939	break;
879	940	case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
	941	+ if (!test_kvm_facility(kvm, 76)) {
	942	+ mutex_unlock(&kvm->lock);
	943	+ return -EINVAL;
	944	+ }
880	945	get_random_bytes(
881	946	kvm->arch.crypto.crycb->dea_wrapping_key_mask,
882	947	sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
..	..	@@ -884,16 +949,38 @@
884	949	VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
885	950	break;
886	951	case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
	952	+ if (!test_kvm_facility(kvm, 76)) {
	953	+ mutex_unlock(&kvm->lock);
	954	+ return -EINVAL;
	955	+ }
887	956	kvm->arch.crypto.aes_kw = 0;
888	957	memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
889	958	sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
890	959	VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
891	960	break;
892	961	case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
	962	+ if (!test_kvm_facility(kvm, 76)) {
	963	+ mutex_unlock(&kvm->lock);
	964	+ return -EINVAL;
	965	+ }
893	966	kvm->arch.crypto.dea_kw = 0;
894	967	memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
895	968	sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
896	969	VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
	970	+ break;
	971	+ case KVM_S390_VM_CRYPTO_ENABLE_APIE:
	972	+ if (!ap_instructions_available()) {
	973	+ mutex_unlock(&kvm->lock);
	974	+ return -EOPNOTSUPP;
	975	+ }
	976	+ kvm->arch.crypto.apie = 1;
	977	+ break;
	978	+ case KVM_S390_VM_CRYPTO_DISABLE_APIE:
	979	+ if (!ap_instructions_available()) {
	980	+ mutex_unlock(&kvm->lock);
	981	+ return -EOPNOTSUPP;
	982	+ }
	983	+ kvm->arch.crypto.apie = 0;
897	984	break;
898	985	default:
899	986	mutex_unlock(&kvm->lock);
..	..	@@ -1005,6 +1092,8 @@
1005	1092	return 0;
1006	1093	}
1007	1094
	1095	+static void __kvm_s390_set_tod_clock(struct kvm kvm, const struct kvm_s390_vm_tod_clock gtod);
	1096	+
1008	1097	static int kvm_s390_set_tod_ext(struct kvm kvm, struct kvm_device_attr attr)
1009	1098	{
1010	1099	struct kvm_s390_vm_tod_clock gtod;
..	..	@@ -1014,7 +1103,7 @@
1014	1103
1015	1104	if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1016	1105	return -EINVAL;
1017		- kvm_s390_set_tod_clock(kvm, &gtod);
	1106	+ __kvm_s390_set_tod_clock(kvm, &gtod);
1018	1107
1019	1108	VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1020	1109	gtod.epoch_idx, gtod.tod);
..	..	@@ -1045,7 +1134,7 @@
1045	1134	sizeof(gtod.tod)))
1046	1135	return -EFAULT;
1047	1136
1048		- kvm_s390_set_tod_clock(kvm, &gtod);
	1137	+ __kvm_s390_set_tod_clock(kvm, &gtod);
1049	1138	VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1050	1139	return 0;
1051	1140	}
..	..	@@ -1056,6 +1145,16 @@
1056	1145
1057	1146	if (attr->flags)
1058	1147	return -EINVAL;
	1148	+
	1149	+ mutex_lock(&kvm->lock);
	1150	+ /*
	1151	+ * For protected guests, the TOD is managed by the ultravisor, so trying
	1152	+ * to change it will never bring the expected results.
	1153	+ */
	1154	+ if (kvm_s390_pv_is_protected(kvm)) {
	1155	+ ret = -EOPNOTSUPP;
	1156	+ goto out_unlock;
	1157	+ }
1059	1158
1060	1159	switch (attr->attr) {
1061	1160	case KVM_S390_VM_TOD_EXT:
..	..	@@ -1071,6 +1170,9 @@
1071	1170	ret = -ENXIO;
1072	1171	break;
1073	1172	}
	1173	+
	1174	+out_unlock:
	1175	+ mutex_unlock(&kvm->lock);
1074	1176	return ret;
1075	1177	}
1076	1178
..	..	@@ -1232,11 +1334,78 @@
1232	1334	static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1233	1335	struct kvm_device_attr *attr)
1234	1336	{
1235		- /*
1236		- * Once supported by kernel + hw, we have to store the subfunctions
1237		- * in kvm->arch and remember that user space configured them.
1238		- */
1239		- return -ENXIO;
	1337	+ mutex_lock(&kvm->lock);
	1338	+ if (kvm->created_vcpus) {
	1339	+ mutex_unlock(&kvm->lock);
	1340	+ return -EBUSY;
	1341	+ }
	1342	+
	1343	+ if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
	1344	+ sizeof(struct kvm_s390_vm_cpu_subfunc))) {
	1345	+ mutex_unlock(&kvm->lock);
	1346	+ return -EFAULT;
	1347	+ }
	1348	+ mutex_unlock(&kvm->lock);
	1349	+
	1350	+ VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
	1351	+ ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
	1352	+ ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
	1353	+ ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
	1354	+ ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
	1355	+ VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
	1356	+ ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
	1357	+ ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
	1358	+ VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
	1359	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
	1360	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
	1361	+ VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
	1362	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
	1363	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
	1364	+ VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
	1365	+ ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
	1366	+ ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
	1367	+ VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
	1368	+ ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
	1369	+ ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
	1370	+ VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
	1371	+ ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
	1372	+ ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
	1373	+ VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
	1374	+ ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
	1375	+ ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
	1376	+ VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
	1377	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
	1378	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
	1379	+ VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
	1380	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
	1381	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
	1382	+ VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
	1383	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
	1384	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
	1385	+ VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
	1386	+ ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
	1387	+ ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
	1388	+ VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
	1389	+ ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
	1390	+ ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
	1391	+ VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
	1392	+ ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
	1393	+ ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
	1394	+ VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
	1395	+ ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
	1396	+ ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
	1397	+ VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
	1398	+ ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
	1399	+ ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
	1400	+ ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
	1401	+ ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
	1402	+ VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
	1403	+ ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
	1404	+ ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
	1405	+ ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
	1406	+ ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
	1407	+
	1408	+ return 0;
1240	1409	}
1241	1410
1242	1411	static int kvm_s390_set_cpu_model(struct kvm kvm, struct kvm_device_attr attr)
..	..	@@ -1355,12 +1524,69 @@
1355	1524	static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1356	1525	struct kvm_device_attr *attr)
1357	1526	{
1358		- /*
1359		- * Once we can actually configure subfunctions (kernel + hw support),
1360		- * we have to check if they were already set by user space, if so copy
1361		- * them from kvm->arch.
1362		- */
1363		- return -ENXIO;
	1527	+ if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
	1528	+ sizeof(struct kvm_s390_vm_cpu_subfunc)))
	1529	+ return -EFAULT;
	1530	+
	1531	+ VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
	1532	+ ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
	1533	+ ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
	1534	+ ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
	1535	+ ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
	1536	+ VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
	1537	+ ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
	1538	+ ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
	1539	+ VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
	1540	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
	1541	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
	1542	+ VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
	1543	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
	1544	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
	1545	+ VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
	1546	+ ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
	1547	+ ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
	1548	+ VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
	1549	+ ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
	1550	+ ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
	1551	+ VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
	1552	+ ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
	1553	+ ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
	1554	+ VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
	1555	+ ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
	1556	+ ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
	1557	+ VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
	1558	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
	1559	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
	1560	+ VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
	1561	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
	1562	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
	1563	+ VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
	1564	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
	1565	+ ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
	1566	+ VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
	1567	+ ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
	1568	+ ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
	1569	+ VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
	1570	+ ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
	1571	+ ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
	1572	+ VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
	1573	+ ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
	1574	+ ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
	1575	+ VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
	1576	+ ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
	1577	+ ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
	1578	+ VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
	1579	+ ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
	1580	+ ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
	1581	+ ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
	1582	+ ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
	1583	+ VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
	1584	+ ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
	1585	+ ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
	1586	+ ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
	1587	+ ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
	1588	+
	1589	+ return 0;
1364	1590	}
1365	1591
1366	1592	static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
..	..	@@ -1369,8 +1595,68 @@
1369	1595	if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1370	1596	sizeof(struct kvm_s390_vm_cpu_subfunc)))
1371	1597	return -EFAULT;
	1598	+
	1599	+ VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
	1600	+ ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
	1601	+ ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
	1602	+ ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
	1603	+ ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
	1604	+ VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
	1605	+ ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
	1606	+ ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
	1607	+ VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
	1608	+ ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
	1609	+ ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
	1610	+ VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
	1611	+ ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
	1612	+ ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
	1613	+ VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
	1614	+ ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
	1615	+ ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
	1616	+ VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
	1617	+ ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
	1618	+ ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
	1619	+ VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
	1620	+ ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
	1621	+ ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
	1622	+ VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
	1623	+ ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
	1624	+ ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
	1625	+ VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
	1626	+ ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
	1627	+ ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
	1628	+ VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
	1629	+ ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
	1630	+ ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
	1631	+ VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
	1632	+ ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
	1633	+ ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
	1634	+ VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
	1635	+ ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
	1636	+ ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
	1637	+ VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
	1638	+ ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
	1639	+ ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
	1640	+ VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
	1641	+ ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
	1642	+ ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
	1643	+ VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
	1644	+ ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
	1645	+ ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
	1646	+ VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
	1647	+ ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
	1648	+ ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
	1649	+ ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
	1650	+ ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
	1651	+ VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
	1652	+ ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
	1653	+ ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
	1654	+ ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
	1655	+ ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
	1656	+
1372	1657	return 0;
1373	1658	}
	1659	+
1374	1660	static int kvm_s390_get_cpu_model(struct kvm kvm, struct kvm_device_attr attr)
1375	1661	{
1376	1662	int ret = -ENXIO;
..	..	@@ -1488,10 +1774,9 @@
1488	1774	case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1489	1775	case KVM_S390_VM_CPU_MACHINE_FEAT:
1490	1776	case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
	1777	+ case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1491	1778	ret = 0;
1492	1779	break;
1493		- /* configuring subfunctions is not supported yet */
1494		- case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1495	1780	default:
1496	1781	ret = -ENXIO;
1497	1782	break;
..	..	@@ -1504,6 +1789,10 @@
1504	1789	case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1505	1790	case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1506	1791	ret = 0;
	1792	+ break;
	1793	+ case KVM_S390_VM_CRYPTO_ENABLE_APIE:
	1794	+ case KVM_S390_VM_CRYPTO_DISABLE_APIE:
	1795	+ ret = ap_instructions_available() ? 0 : -ENXIO;
1507	1796	break;
1508	1797	default:
1509	1798	ret = -ENXIO;
..	..	@@ -1542,7 +1831,7 @@
1542	1831	if (!keys)
1543	1832	return -ENOMEM;
1544	1833
1545		- down_read(&current->mm->mmap_sem);
	1834	+ mmap_read_lock(current->mm);
1546	1835	srcu_idx = srcu_read_lock(&kvm->srcu);
1547	1836	for (i = 0; i < args->count; i++) {
1548	1837	hva = gfn_to_hva(kvm, args->start_gfn + i);
..	..	@@ -1556,7 +1845,7 @@
1556	1845	break;
1557	1846	}
1558	1847	srcu_read_unlock(&kvm->srcu, srcu_idx);
1559		- up_read(&current->mm->mmap_sem);
	1848	+ mmap_read_unlock(current->mm);
1560	1849
1561	1850	if (!r) {
1562	1851	r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
..	..	@@ -1600,7 +1889,7 @@
1600	1889	goto out;
1601	1890
1602	1891	i = 0;
1603		- down_read(&current->mm->mmap_sem);
	1892	+ mmap_read_lock(current->mm);
1604	1893	srcu_idx = srcu_read_lock(&kvm->srcu);
1605	1894	while (i < args->count) {
1606	1895	unlocked = false;
..	..	@@ -1618,7 +1907,7 @@
1618	1907
1619	1908	r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1620	1909	if (r) {
1621		- r = fixup_user_fault(current, current->mm, hva,
	1910	+ r = fixup_user_fault(current->mm, hva,
1622	1911	FAULT_FLAG_WRITE, &unlocked);
1623	1912	if (r)
1624	1913	break;
..	..	@@ -1627,7 +1916,7 @@
1627	1916	i++;
1628	1917	}
1629	1918	srcu_read_unlock(&kvm->srcu, srcu_idx);
1630		- up_read(&current->mm->mmap_sem);
	1919	+ mmap_read_unlock(current->mm);
1631	1920	out:
1632	1921	kvfree(keys);
1633	1922	return r;
..	..	@@ -1732,6 +2021,9 @@
1732	2021	struct kvm_memslots *slots = kvm_memslots(kvm);
1733	2022	struct kvm_memory_slot *ms;
1734	2023
	2024	+ if (unlikely(!slots->used_slots))
	2025	+ return 0;
	2026	+
1735	2027	cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
1736	2028	ms = gfn_to_memslot(kvm, cur_gfn);
1737	2029	args->count = 0;
..	..	@@ -1813,14 +2105,14 @@
1813	2105	if (!values)
1814	2106	return -ENOMEM;
1815	2107
1816		- down_read(&kvm->mm->mmap_sem);
	2108	+ mmap_read_lock(kvm->mm);
1817	2109	srcu_idx = srcu_read_lock(&kvm->srcu);
1818	2110	if (peek)
1819	2111	ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
1820	2112	else
1821	2113	ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
1822	2114	srcu_read_unlock(&kvm->srcu, srcu_idx);
1823		- up_read(&kvm->mm->mmap_sem);
	2115	+ mmap_read_unlock(kvm->mm);
1824	2116
1825	2117	if (kvm->arch.migration_mode)
1826	2118	args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
..	..	@@ -1870,7 +2162,7 @@
1870	2162	goto out;
1871	2163	}
1872	2164
1873		- down_read(&kvm->mm->mmap_sem);
	2165	+ mmap_read_lock(kvm->mm);
1874	2166	srcu_idx = srcu_read_lock(&kvm->srcu);
1875	2167	for (i = 0; i < args->count; i++) {
1876	2168	hva = gfn_to_hva(kvm, args->start_gfn + i);
..	..	@@ -1885,15 +2177,203 @@
1885	2177	set_pgste_bits(kvm->mm, hva, mask, pgstev);
1886	2178	}
1887	2179	srcu_read_unlock(&kvm->srcu, srcu_idx);
1888		- up_read(&kvm->mm->mmap_sem);
	2180	+ mmap_read_unlock(kvm->mm);
1889	2181
1890	2182	if (!kvm->mm->context.uses_cmm) {
1891		- down_write(&kvm->mm->mmap_sem);
	2183	+ mmap_write_lock(kvm->mm);
1892	2184	kvm->mm->context.uses_cmm = 1;
1893		- up_write(&kvm->mm->mmap_sem);
	2185	+ mmap_write_unlock(kvm->mm);
1894	2186	}
1895	2187	out:
1896	2188	vfree(bits);
	2189	+ return r;
	2190	+}
	2191	+
	2192	+static int kvm_s390_cpus_from_pv(struct kvm kvm, u16 rcp, u16 *rrcp)
	2193	+{
	2194	+ struct kvm_vcpu *vcpu;
	2195	+ u16 rc, rrc;
	2196	+ int ret = 0;
	2197	+ int i;
	2198	+
	2199	+ /*
	2200	+ * We ignore failures and try to destroy as many CPUs as possible.
	2201	+ * At the same time we must not free the assigned resources when
	2202	+ * this fails, as the ultravisor has still access to that memory.
	2203	+ * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
	2204	+ * behind.
	2205	+ * We want to return the first failure rc and rrc, though.
	2206	+ */
	2207	+ kvm_for_each_vcpu(i, vcpu, kvm) {
	2208	+ mutex_lock(&vcpu->mutex);
	2209	+ if (kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc) && !ret) {
	2210	+ *rcp = rc;
	2211	+ *rrcp = rrc;
	2212	+ ret = -EIO;
	2213	+ }
	2214	+ mutex_unlock(&vcpu->mutex);
	2215	+ }
	2216	+ return ret;
	2217	+}
	2218	+
	2219	+static int kvm_s390_cpus_to_pv(struct kvm kvm, u16 rc, u16 *rrc)
	2220	+{
	2221	+ int i, r = 0;
	2222	+ u16 dummy;
	2223	+
	2224	+ struct kvm_vcpu *vcpu;
	2225	+
	2226	+ kvm_for_each_vcpu(i, vcpu, kvm) {
	2227	+ mutex_lock(&vcpu->mutex);
	2228	+ r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
	2229	+ mutex_unlock(&vcpu->mutex);
	2230	+ if (r)
	2231	+ break;
	2232	+ }
	2233	+ if (r)
	2234	+ kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
	2235	+ return r;
	2236	+}
	2237	+
	2238	+static int kvm_s390_handle_pv(struct kvm kvm, struct kvm_pv_cmd cmd)
	2239	+{
	2240	+ int r = 0;
	2241	+ u16 dummy;
	2242	+ void __user argp = (void __user )cmd->data;
	2243	+
	2244	+ switch (cmd->cmd) {
	2245	+ case KVM_PV_ENABLE: {
	2246	+ r = -EINVAL;
	2247	+ if (kvm_s390_pv_is_protected(kvm))
	2248	+ break;
	2249	+
	2250	+ /*
	2251	+ * FMT 4 SIE needs esca. As we never switch back to bsca from
	2252	+ * esca, we need no cleanup in the error cases below
	2253	+ */
	2254	+ r = sca_switch_to_extended(kvm);
	2255	+ if (r)
	2256	+ break;
	2257	+
	2258	+ mmap_write_lock(current->mm);
	2259	+ r = gmap_mark_unmergeable();
	2260	+ mmap_write_unlock(current->mm);
	2261	+ if (r)
	2262	+ break;
	2263	+
	2264	+ r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
	2265	+ if (r)
	2266	+ break;
	2267	+
	2268	+ r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
	2269	+ if (r)
	2270	+ kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
	2271	+
	2272	+ /* we need to block service interrupts from now on */
	2273	+ set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
	2274	+ break;
	2275	+ }
	2276	+ case KVM_PV_DISABLE: {
	2277	+ r = -EINVAL;
	2278	+ if (!kvm_s390_pv_is_protected(kvm))
	2279	+ break;
	2280	+
	2281	+ r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
	2282	+ /*
	2283	+ * If a CPU could not be destroyed, destroy VM will also fail.
	2284	+ * There is no point in trying to destroy it. Instead return
	2285	+ * the rc and rrc from the first CPU that failed destroying.
	2286	+ */
	2287	+ if (r)
	2288	+ break;
	2289	+ r = kvm_s390_pv_deinit_vm(kvm, &cmd->rc, &cmd->rrc);
	2290	+
	2291	+ /* no need to block service interrupts any more */
	2292	+ clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
	2293	+ break;
	2294	+ }
	2295	+ case KVM_PV_SET_SEC_PARMS: {
	2296	+ struct kvm_s390_pv_sec_parm parms = {};
	2297	+ void *hdr;
	2298	+
	2299	+ r = -EINVAL;
	2300	+ if (!kvm_s390_pv_is_protected(kvm))
	2301	+ break;
	2302	+
	2303	+ r = -EFAULT;
	2304	+ if (copy_from_user(&parms, argp, sizeof(parms)))
	2305	+ break;
	2306	+
	2307	+ /* Currently restricted to 8KB */
	2308	+ r = -EINVAL;
	2309	+ if (parms.length > PAGE_SIZE * 2)
	2310	+ break;
	2311	+
	2312	+ r = -ENOMEM;
	2313	+ hdr = vmalloc(parms.length);
	2314	+ if (!hdr)
	2315	+ break;
	2316	+
	2317	+ r = -EFAULT;
	2318	+ if (!copy_from_user(hdr, (void __user *)parms.origin,
	2319	+ parms.length))
	2320	+ r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
	2321	+ &cmd->rc, &cmd->rrc);
	2322	+
	2323	+ vfree(hdr);
	2324	+ break;
	2325	+ }
	2326	+ case KVM_PV_UNPACK: {
	2327	+ struct kvm_s390_pv_unp unp = {};
	2328	+
	2329	+ r = -EINVAL;
	2330	+ if (!kvm_s390_pv_is_protected(kvm) \|\| !mm_is_protected(kvm->mm))
	2331	+ break;
	2332	+
	2333	+ r = -EFAULT;
	2334	+ if (copy_from_user(&unp, argp, sizeof(unp)))
	2335	+ break;
	2336	+
	2337	+ r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
	2338	+ &cmd->rc, &cmd->rrc);
	2339	+ break;
	2340	+ }
	2341	+ case KVM_PV_VERIFY: {
	2342	+ r = -EINVAL;
	2343	+ if (!kvm_s390_pv_is_protected(kvm))
	2344	+ break;
	2345	+
	2346	+ r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
	2347	+ UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
	2348	+ KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
	2349	+ cmd->rrc);
	2350	+ break;
	2351	+ }
	2352	+ case KVM_PV_PREP_RESET: {
	2353	+ r = -EINVAL;
	2354	+ if (!kvm_s390_pv_is_protected(kvm))
	2355	+ break;
	2356	+
	2357	+ r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
	2358	+ UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
	2359	+ KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
	2360	+ cmd->rc, cmd->rrc);
	2361	+ break;
	2362	+ }
	2363	+ case KVM_PV_UNSHARE_ALL: {
	2364	+ r = -EINVAL;
	2365	+ if (!kvm_s390_pv_is_protected(kvm))
	2366	+ break;
	2367	+
	2368	+ r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
	2369	+ UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
	2370	+ KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
	2371	+ cmd->rc, cmd->rrc);
	2372	+ break;
	2373	+ }
	2374	+ default:
	2375	+ r = -ENOTTY;
	2376	+ }
1897	2377	return r;
1898	2378	}
1899	2379
..	..	@@ -1913,14 +2393,6 @@
1913	2393	if (copy_from_user(&s390int, argp, sizeof(s390int)))
1914	2394	break;
1915	2395	r = kvm_s390_inject_vm(kvm, &s390int);
1916		- break;
1917		- }
1918		- case KVM_ENABLE_CAP: {
1919		- struct kvm_enable_cap cap;
1920		- r = -EFAULT;
1921		- if (copy_from_user(&cap, argp, sizeof(cap)))
1922		- break;
1923		- r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1924	2396	break;
1925	2397	}
1926	2398	case KVM_CREATE_IRQCHIP: {
..	..	@@ -2002,6 +2474,33 @@
2002	2474	mutex_unlock(&kvm->slots_lock);
2003	2475	break;
2004	2476	}
	2477	+ case KVM_S390_PV_COMMAND: {
	2478	+ struct kvm_pv_cmd args;
	2479	+
	2480	+ /* protvirt means user sigp */
	2481	+ kvm->arch.user_cpu_state_ctrl = 1;
	2482	+ r = 0;
	2483	+ if (!is_prot_virt_host()) {
	2484	+ r = -EINVAL;
	2485	+ break;
	2486	+ }
	2487	+ if (copy_from_user(&args, argp, sizeof(args))) {
	2488	+ r = -EFAULT;
	2489	+ break;
	2490	+ }
	2491	+ if (args.flags) {
	2492	+ r = -EINVAL;
	2493	+ break;
	2494	+ }
	2495	+ mutex_lock(&kvm->lock);
	2496	+ r = kvm_s390_handle_pv(kvm, &args);
	2497	+ mutex_unlock(&kvm->lock);
	2498	+ if (copy_to_user(argp, &args, sizeof(args))) {
	2499	+ r = -EFAULT;
	2500	+ break;
	2501	+ }
	2502	+ break;
	2503	+ }
2005	2504	default:
2006	2505	r = -ENOTTY;
2007	2506	}
..	..	@@ -2009,54 +2508,100 @@
2009	2508	return r;
2010	2509	}
2011	2510
2012		-static int kvm_s390_query_ap_config(u8 *config)
2013		-{
2014		- u32 fcn_code = 0x04000000UL;
2015		- u32 cc = 0;
2016		-
2017		- memset(config, 0, 128);
2018		- asm volatile(
2019		- "lgr 0,%1\n"
2020		- "lgr 2,%2\n"
2021		- ".long 0xb2af0000\n" /* PQAP(QCI) */
2022		- "0: ipm %0\n"
2023		- "srl %0,28\n"
2024		- "1:\n"
2025		- EX_TABLE(0b, 1b)
2026		- : "+r" (cc)
2027		- : "r" (fcn_code), "r" (config)
2028		- : "cc", "0", "2", "memory"
2029		- );
2030		-
2031		- return cc;
2032		-}
2033		-
2034	2511	static int kvm_s390_apxa_installed(void)
2035	2512	{
2036		- u8 config[128];
2037		- int cc;
	2513	+ struct ap_config_info info;
2038	2514
2039		- if (test_facility(12)) {
2040		- cc = kvm_s390_query_ap_config(config);
2041		-
2042		- if (cc)
2043		- pr_err("PQAP(QCI) failed with cc=%d", cc);
2044		- else
2045		- return config[0] & 0x40;
	2515	+ if (ap_instructions_available()) {
	2516	+ if (ap_qci(&info) == 0)
	2517	+ return info.apxa;
2046	2518	}
2047	2519
2048	2520	return 0;
2049	2521	}
2050	2522
	2523	+/*
	2524	+ * The format of the crypto control block (CRYCB) is specified in the 3 low
	2525	+ * order bits of the CRYCB designation (CRYCBD) field as follows:
	2526	+ * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
	2527	+ * AP extended addressing (APXA) facility are installed.
	2528	+ * Format 1: The APXA facility is not installed but the MSAX3 facility is.
	2529	+ * Format 2: Both the APXA and MSAX3 facilities are installed
	2530	+ */
2051	2531	static void kvm_s390_set_crycb_format(struct kvm *kvm)
2052	2532	{
2053	2533	kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
	2534	+
	2535	+ /* Clear the CRYCB format bits - i.e., set format 0 by default */
	2536	+ kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
	2537	+
	2538	+ /* Check whether MSAX3 is installed */
	2539	+ if (!test_kvm_facility(kvm, 76))
	2540	+ return;
2054	2541
2055	2542	if (kvm_s390_apxa_installed())
2056	2543	kvm->arch.crypto.crycbd \|= CRYCB_FORMAT2;
2057	2544	else
2058	2545	kvm->arch.crypto.crycbd \|= CRYCB_FORMAT1;
2059	2546	}
	2547	+
	2548	+void kvm_arch_crypto_set_masks(struct kvm kvm, unsigned long apm,
	2549	+ unsigned long aqm, unsigned long adm)
	2550	+{
	2551	+ struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
	2552	+
	2553	+ mutex_lock(&kvm->lock);
	2554	+ kvm_s390_vcpu_block_all(kvm);
	2555	+
	2556	+ switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
	2557	+ case CRYCB_FORMAT2: /* APCB1 use 256 bits */
	2558	+ memcpy(crycb->apcb1.apm, apm, 32);
	2559	+ VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
	2560	+ apm[0], apm[1], apm[2], apm[3]);
	2561	+ memcpy(crycb->apcb1.aqm, aqm, 32);
	2562	+ VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
	2563	+ aqm[0], aqm[1], aqm[2], aqm[3]);
	2564	+ memcpy(crycb->apcb1.adm, adm, 32);
	2565	+ VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
	2566	+ adm[0], adm[1], adm[2], adm[3]);
	2567	+ break;
	2568	+ case CRYCB_FORMAT1:
	2569	+ case CRYCB_FORMAT0: /* Fall through both use APCB0 */
	2570	+ memcpy(crycb->apcb0.apm, apm, 8);
	2571	+ memcpy(crycb->apcb0.aqm, aqm, 2);
	2572	+ memcpy(crycb->apcb0.adm, adm, 2);
	2573	+ VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
	2574	+ apm[0], ((unsigned short )aqm),
	2575	+ ((unsigned short )adm));
	2576	+ break;
	2577	+ default: /* Can not happen */
	2578	+ break;
	2579	+ }
	2580	+
	2581	+ /* recreate the shadow crycb for each vcpu */
	2582	+ kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
	2583	+ kvm_s390_vcpu_unblock_all(kvm);
	2584	+ mutex_unlock(&kvm->lock);
	2585	+}
	2586	+EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
	2587	+
	2588	+void kvm_arch_crypto_clear_masks(struct kvm *kvm)
	2589	+{
	2590	+ mutex_lock(&kvm->lock);
	2591	+ kvm_s390_vcpu_block_all(kvm);
	2592	+
	2593	+ memset(&kvm->arch.crypto.crycb->apcb0, 0,
	2594	+ sizeof(kvm->arch.crypto.crycb->apcb0));
	2595	+ memset(&kvm->arch.crypto.crycb->apcb1, 0,
	2596	+ sizeof(kvm->arch.crypto.crycb->apcb1));
	2597	+
	2598	+ VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
	2599	+ /* recreate the shadow crycb for each vcpu */
	2600	+ kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
	2601	+ kvm_s390_vcpu_unblock_all(kvm);
	2602	+ mutex_unlock(&kvm->lock);
	2603	+}
	2604	+EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2060	2605
2061	2606	static u64 kvm_s390_get_initial_cpuid(void)
2062	2607	{
..	..	@@ -2069,11 +2614,11 @@
2069	2614
2070	2615	static void kvm_s390_crypto_init(struct kvm *kvm)
2071	2616	{
2072		- if (!test_kvm_facility(kvm, 76))
2073		- return;
2074		-
2075	2617	kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2076	2618	kvm_s390_set_crycb_format(kvm);
	2619	+
	2620	+ if (!test_kvm_facility(kvm, 76))
	2621	+ return;
2077	2622
2078	2623	/* Enable AES/DEA protected key functions by default */
2079	2624	kvm->arch.crypto.aes_kw = 1;
..	..	@@ -2144,6 +2689,7 @@
2144	2689	if (!kvm->arch.sie_page2)
2145	2690	goto out_err;
2146	2691
	2692	+ kvm->arch.sie_page2->kvm = kvm;
2147	2693	kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2148	2694
2149	2695	for (i = 0; i < kvm_s390_fac_size(); i++) {
..	..	@@ -2153,6 +2699,7 @@
2153	2699	kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2154	2700	kvm_s390_fac_base[i];
2155	2701	}
	2702	+ kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
2156	2703
2157	2704	/* we are always in czam mode - even on pre z14 machines */
2158	2705	set_kvm_facility(kvm->arch.model.fac_mask, 138);
..	..	@@ -2164,6 +2711,9 @@
2164	2711	set_kvm_facility(kvm->arch.model.fac_mask, 147);
2165	2712	set_kvm_facility(kvm->arch.model.fac_list, 147);
2166	2713	}
	2714	+
	2715	+ if (css_general_characteristics.aiv && test_facility(65))
	2716	+ set_kvm_facility(kvm->arch.model.fac_mask, 65);
2167	2717
2168	2718	kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2169	2719	kvm->arch.model.ibc = sclp.ibc & 0x0fff;
..	..	@@ -2200,7 +2750,8 @@
2200	2750	kvm->arch.use_skf = sclp.has_skey;
2201	2751	spin_lock_init(&kvm->arch.start_stop_lock);
2202	2752	kvm_s390_vsie_init(kvm);
2203		- kvm_s390_gisa_init(kvm);
	2753	+ if (use_gisa)
	2754	+ kvm_s390_gisa_init(kvm);
2204	2755	KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2205	2756
2206	2757	return 0;
..	..	@@ -2212,18 +2763,10 @@
2212	2763	return rc;
2213	2764	}
2214	2765
2215		-bool kvm_arch_has_vcpu_debugfs(void)
2216		-{
2217		- return false;
2218		-}
2219		-
2220		-int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2221		-{
2222		- return 0;
2223		-}
2224		-
2225	2766	void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2226	2767	{
	2768	+ u16 rc, rrc;
	2769	+
2227	2770	VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2228	2771	trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2229	2772	kvm_s390_clear_local_irqs(vcpu);
..	..	@@ -2236,10 +2779,10 @@
2236	2779
2237	2780	if (vcpu->kvm->arch.use_cmma)
2238	2781	kvm_s390_vcpu_unsetup_cmma(vcpu);
	2782	+ /* We can not hold the vcpu mutex here, we are already dying */
	2783	+ if (kvm_s390_pv_cpu_get_handle(vcpu))
	2784	+ kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
2239	2785	free_page((unsigned long)(vcpu->arch.sie_block));
2240		-
2241		- kvm_vcpu_uninit(vcpu);
2242		- kmem_cache_free(kvm_vcpu_cache, vcpu);
2243	2786	}
2244	2787
2245	2788	static void kvm_free_vcpus(struct kvm *kvm)
..	..	@@ -2248,7 +2791,7 @@
2248	2791	struct kvm_vcpu *vcpu;
2249	2792
2250	2793	kvm_for_each_vcpu(i, vcpu, kvm)
2251		- kvm_arch_vcpu_destroy(vcpu);
	2794	+ kvm_vcpu_destroy(vcpu);
2252	2795
2253	2796	mutex_lock(&kvm->lock);
2254	2797	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
..	..	@@ -2260,10 +2803,20 @@
2260	2803
2261	2804	void kvm_arch_destroy_vm(struct kvm *kvm)
2262	2805	{
	2806	+ u16 rc, rrc;
	2807	+
2263	2808	kvm_free_vcpus(kvm);
2264	2809	sca_dispose(kvm);
2265		- debug_unregister(kvm->arch.dbf);
2266	2810	kvm_s390_gisa_destroy(kvm);
	2811	+ /*
	2812	+ * We are already at the end of life and kvm->lock is not taken.
	2813	+ * This is ok as the file descriptor is closed by now and nobody
	2814	+ * can mess with the pv state. To avoid lockdep_assert_held from
	2815	+ * complaining we do not use kvm_s390_pv_is_protected.
	2816	+ */
	2817	+ if (kvm_s390_pv_get_handle(kvm))
	2818	+ kvm_s390_pv_deinit_vm(kvm, &rc, &rrc);
	2819	+ debug_unregister(kvm->arch.dbf);
2267	2820	free_page((unsigned long)kvm->arch.sie_page2);
2268	2821	if (!kvm_is_ucontrol(kvm))
2269	2822	gmap_remove(kvm->arch.gmap);
..	..	@@ -2359,6 +2912,9 @@
2359	2912	unsigned int vcpu_idx;
2360	2913	u32 scaol, scaoh;
2361	2914
	2915	+ if (kvm->arch.use_esca)
	2916	+ return 0;
	2917	+
2362	2918	new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL\|__GFP_ZERO);
2363	2919	if (!new_sca)
2364	2920	return -ENOMEM;
..	..	@@ -2408,39 +2964,6 @@
2408	2964	mutex_unlock(&kvm->lock);
2409	2965
2410	2966	return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2411		-}
2412		-
2413		-int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2414		-{
2415		- vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2416		- kvm_clear_async_pf_completion_queue(vcpu);
2417		- vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX \|
2418		- KVM_SYNC_GPRS \|
2419		- KVM_SYNC_ACRS \|
2420		- KVM_SYNC_CRS \|
2421		- KVM_SYNC_ARCH0 \|
2422		- KVM_SYNC_PFAULT;
2423		- kvm_s390_set_prefix(vcpu, 0);
2424		- if (test_kvm_facility(vcpu->kvm, 64))
2425		- vcpu->run->kvm_valid_regs \|= KVM_SYNC_RICCB;
2426		- if (test_kvm_facility(vcpu->kvm, 82))
2427		- vcpu->run->kvm_valid_regs \|= KVM_SYNC_BPBC;
2428		- if (test_kvm_facility(vcpu->kvm, 133))
2429		- vcpu->run->kvm_valid_regs \|= KVM_SYNC_GSCB;
2430		- if (test_kvm_facility(vcpu->kvm, 156))
2431		- vcpu->run->kvm_valid_regs \|= KVM_SYNC_ETOKEN;
2432		- /* fprs can be synchronized via vrs, even if the guest has no vx. With
2433		- * MACHINE_HAS_VX, (load\|store)_fpu_regs() will work with vrs format.
2434		- */
2435		- if (MACHINE_HAS_VX)
2436		- vcpu->run->kvm_valid_regs \|= KVM_SYNC_VRS;
2437		- else
2438		- vcpu->run->kvm_valid_regs \|= KVM_SYNC_FPRS;
2439		-
2440		- if (kvm_is_ucontrol(vcpu->kvm))
2441		- return __kvm_ucontrol_vcpu_init(vcpu);
2442		-
2443		- return 0;
2444	2967	}
2445	2968
2446	2969	/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
..	..	@@ -2551,33 +3074,6 @@
2551	3074
2552	3075	}
2553	3076
2554		-static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2555		-{
2556		- /* this equals initial cpu reset in pop, but we don't switch to ESA */
2557		- vcpu->arch.sie_block->gpsw.mask = 0UL;
2558		- vcpu->arch.sie_block->gpsw.addr = 0UL;
2559		- kvm_s390_set_prefix(vcpu, 0);
2560		- kvm_s390_set_cpu_timer(vcpu, 0);
2561		- vcpu->arch.sie_block->ckc = 0UL;
2562		- vcpu->arch.sie_block->todpr = 0;
2563		- memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2564		- vcpu->arch.sie_block->gcr[0] = CR0_UNUSED_56 \|
2565		- CR0_INTERRUPT_KEY_SUBMASK \|
2566		- CR0_MEASUREMENT_ALERT_SUBMASK;
2567		- vcpu->arch.sie_block->gcr[14] = CR14_UNUSED_32 \|
2568		- CR14_UNUSED_33 \|
2569		- CR14_EXTERNAL_DAMAGE_SUBMASK;
2570		- vcpu->run->s.regs.fpc = 0;
2571		- vcpu->arch.sie_block->gbea = 1;
2572		- vcpu->arch.sie_block->pp = 0;
2573		- vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
2574		- vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2575		- kvm_clear_async_pf_completion_queue(vcpu);
2576		- if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2577		- kvm_s390_vcpu_stop(vcpu);
2578		- kvm_s390_clear_local_irqs(vcpu);
2579		-}
2580		-
2581	3077	void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2582	3078	{
2583	3079	mutex_lock(&vcpu->kvm->lock);
..	..	@@ -2596,19 +3092,52 @@
2596	3092	vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2597	3093	}
2598	3094
	3095	+static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
	3096	+{
	3097	+ if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
	3098	+ test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
	3099	+ return true;
	3100	+ return false;
	3101	+}
	3102	+
	3103	+static bool kvm_has_pckmo_ecc(struct kvm *kvm)
	3104	+{
	3105	+ /* At least one ECC subfunction must be present */
	3106	+ return kvm_has_pckmo_subfunc(kvm, 32) \|\|
	3107	+ kvm_has_pckmo_subfunc(kvm, 33) \|\|
	3108	+ kvm_has_pckmo_subfunc(kvm, 34) \|\|
	3109	+ kvm_has_pckmo_subfunc(kvm, 40) \|\|
	3110	+ kvm_has_pckmo_subfunc(kvm, 41);
	3111	+
	3112	+}
	3113	+
2599	3114	static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2600	3115	{
2601		- if (!test_kvm_facility(vcpu->kvm, 76))
	3116	+ /*
	3117	+ * If the AP instructions are not being interpreted and the MSAX3
	3118	+ * facility is not configured for the guest, there is nothing to set up.
	3119	+ */
	3120	+ if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
2602	3121	return;
2603	3122
	3123	+ vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2604	3124	vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES \| ECB3_DEA);
	3125	+ vcpu->arch.sie_block->eca &= ~ECA_APIE;
	3126	+ vcpu->arch.sie_block->ecd &= ~ECD_ECC;
2605	3127
2606		- if (vcpu->kvm->arch.crypto.aes_kw)
	3128	+ if (vcpu->kvm->arch.crypto.apie)
	3129	+ vcpu->arch.sie_block->eca \|= ECA_APIE;
	3130	+
	3131	+ /* Set up protected key support */
	3132	+ if (vcpu->kvm->arch.crypto.aes_kw) {
2607	3133	vcpu->arch.sie_block->ecb3 \|= ECB3_AES;
	3134	+ /* ecc is also wrapped with AES key */
	3135	+ if (kvm_has_pckmo_ecc(vcpu->kvm))
	3136	+ vcpu->arch.sie_block->ecd \|= ECD_ECC;
	3137	+ }
	3138	+
2608	3139	if (vcpu->kvm->arch.crypto.dea_kw)
2609	3140	vcpu->arch.sie_block->ecb3 \|= ECB3_DEA;
2610		-
2611		- vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2612	3141	}
2613	3142
2614	3143	void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
..	..	@@ -2634,9 +3163,10 @@
2634	3163	vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2635	3164	}
2636	3165
2637		-int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
	3166	+static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
2638	3167	{
2639	3168	int rc = 0;
	3169	+ u16 uvrc, uvrrc;
2640	3170
2641	3171	atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH \|
2642	3172	CPUSTAT_SM \|
..	..	@@ -2700,31 +3230,37 @@
2700	3230	hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2701	3231	vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2702	3232
	3233	+ vcpu->arch.sie_block->hpid = HPID_KVM;
	3234	+
2703	3235	kvm_s390_vcpu_crypto_setup(vcpu);
	3236	+
	3237	+ mutex_lock(&vcpu->kvm->lock);
	3238	+ if (kvm_s390_pv_is_protected(vcpu->kvm)) {
	3239	+ rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
	3240	+ if (rc)
	3241	+ kvm_s390_vcpu_unsetup_cmma(vcpu);
	3242	+ }
	3243	+ mutex_unlock(&vcpu->kvm->lock);
2704	3244
2705	3245	return rc;
2706	3246	}
2707	3247
2708		-struct kvm_vcpu kvm_arch_vcpu_create(struct kvm kvm,
2709		- unsigned int id)
	3248	+int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
2710	3249	{
2711		- struct kvm_vcpu *vcpu;
2712		- struct sie_page *sie_page;
2713		- int rc = -EINVAL;
2714		-
2715	3250	if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2716		- goto out;
	3251	+ return -EINVAL;
	3252	+ return 0;
	3253	+}
2717	3254
2718		- rc = -ENOMEM;
2719		-
2720		- vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2721		- if (!vcpu)
2722		- goto out;
	3255	+int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
	3256	+{
	3257	+ struct sie_page *sie_page;
	3258	+ int rc;
2723	3259
2724	3260	BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
2725	3261	sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2726	3262	if (!sie_page)
2727		- goto out_free_cpu;
	3263	+ return -ENOMEM;
2728	3264
2729	3265	vcpu->arch.sie_block = &sie_page->sie_block;
2730	3266	vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
..	..	@@ -2733,31 +3269,65 @@
2733	3269	vcpu->arch.sie_block->mso = 0;
2734	3270	vcpu->arch.sie_block->msl = sclp.hamax;
2735	3271
2736		- vcpu->arch.sie_block->icpua = id;
	3272	+ vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
2737	3273	spin_lock_init(&vcpu->arch.local_int.lock);
2738		- vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa;
	3274	+ vcpu->arch.sie_block->gd = (u32)(u64)vcpu->kvm->arch.gisa_int.origin;
2739	3275	if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
2740	3276	vcpu->arch.sie_block->gd \|= GISA_FORMAT1;
2741	3277	seqcount_init(&vcpu->arch.cputm_seqcount);
2742	3278
2743		- rc = kvm_vcpu_init(vcpu, kvm, id);
2744		- if (rc)
2745		- goto out_free_sie_block;
2746		- VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2747		- vcpu->arch.sie_block);
2748		- trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
	3279	+ vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
	3280	+ kvm_clear_async_pf_completion_queue(vcpu);
	3281	+ vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX \|
	3282	+ KVM_SYNC_GPRS \|
	3283	+ KVM_SYNC_ACRS \|
	3284	+ KVM_SYNC_CRS \|
	3285	+ KVM_SYNC_ARCH0 \|
	3286	+ KVM_SYNC_PFAULT \|
	3287	+ KVM_SYNC_DIAG318;
	3288	+ kvm_s390_set_prefix(vcpu, 0);
	3289	+ if (test_kvm_facility(vcpu->kvm, 64))
	3290	+ vcpu->run->kvm_valid_regs \|= KVM_SYNC_RICCB;
	3291	+ if (test_kvm_facility(vcpu->kvm, 82))
	3292	+ vcpu->run->kvm_valid_regs \|= KVM_SYNC_BPBC;
	3293	+ if (test_kvm_facility(vcpu->kvm, 133))
	3294	+ vcpu->run->kvm_valid_regs \|= KVM_SYNC_GSCB;
	3295	+ if (test_kvm_facility(vcpu->kvm, 156))
	3296	+ vcpu->run->kvm_valid_regs \|= KVM_SYNC_ETOKEN;
	3297	+ /* fprs can be synchronized via vrs, even if the guest has no vx. With
	3298	+ * MACHINE_HAS_VX, (load\|store)_fpu_regs() will work with vrs format.
	3299	+ */
	3300	+ if (MACHINE_HAS_VX)
	3301	+ vcpu->run->kvm_valid_regs \|= KVM_SYNC_VRS;
	3302	+ else
	3303	+ vcpu->run->kvm_valid_regs \|= KVM_SYNC_FPRS;
2749	3304
2750		- return vcpu;
	3305	+ if (kvm_is_ucontrol(vcpu->kvm)) {
	3306	+ rc = __kvm_ucontrol_vcpu_init(vcpu);
	3307	+ if (rc)
	3308	+ goto out_free_sie_block;
	3309	+ }
	3310	+
	3311	+ VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK",
	3312	+ vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
	3313	+ trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
	3314	+
	3315	+ rc = kvm_s390_vcpu_setup(vcpu);
	3316	+ if (rc)
	3317	+ goto out_ucontrol_uninit;
	3318	+ return 0;
	3319	+
	3320	+out_ucontrol_uninit:
	3321	+ if (kvm_is_ucontrol(vcpu->kvm))
	3322	+ gmap_remove(vcpu->arch.gmap);
2751	3323	out_free_sie_block:
2752	3324	free_page((unsigned long)(vcpu->arch.sie_block));
2753		-out_free_cpu:
2754		- kmem_cache_free(kvm_vcpu_cache, vcpu);
2755		-out:
2756		- return ERR_PTR(rc);
	3325	+ return rc;
2757	3326	}
2758	3327
2759	3328	int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2760	3329	{
	3330	+ clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
2761	3331	return kvm_s390_vcpu_has_irq(vcpu, 0);
2762	3332	}
2763	3333
..	..	@@ -2783,18 +3353,25 @@
2783	3353	exit_sie(vcpu);
2784	3354	}
2785	3355
	3356	+bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
	3357	+{
	3358	+ return atomic_read(&vcpu->arch.sie_block->prog20) &
	3359	+ (PROG_BLOCK_SIE \| PROG_REQUEST);
	3360	+}
	3361	+
2786	3362	static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2787	3363	{
2788	3364	atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2789	3365	}
2790	3366
2791	3367	/*
2792		- * Kick a guest cpu out of SIE and wait until SIE is not running.
	3368	+ * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
2793	3369	* If the CPU is not running (e.g. waiting as idle) the function will
2794	3370	* return immediately. */
2795	3371	void exit_sie(struct kvm_vcpu *vcpu)
2796	3372	{
2797	3373	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
	3374	+ kvm_s390_vsie_kick(vcpu);
2798	3375	while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2799	3376	cpu_relax();
2800	3377	}
..	..	@@ -2828,6 +3405,17 @@
2828	3405	kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2829	3406	}
2830	3407	}
	3408	+}
	3409	+
	3410	+bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
	3411	+{
	3412	+ /* do not poll with more than halt_poll_max_steal percent of steal time */
	3413	+ if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
	3414	+ halt_poll_max_steal) {
	3415	+ vcpu->stat.halt_no_poll_steal++;
	3416	+ return true;
	3417	+ }
	3418	+ return false;
2831	3419	}
2832	3420
2833	3421	int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
..	..	@@ -2939,10 +3527,76 @@
2939	3527	return r;
2940	3528	}
2941	3529
2942		-static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
	3530	+static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
2943	3531	{
2944		- kvm_s390_vcpu_initial_reset(vcpu);
2945		- return 0;
	3532	+ vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
	3533	+ vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
	3534	+ memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
	3535	+
	3536	+ kvm_clear_async_pf_completion_queue(vcpu);
	3537	+ if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
	3538	+ kvm_s390_vcpu_stop(vcpu);
	3539	+ kvm_s390_clear_local_irqs(vcpu);
	3540	+}
	3541	+
	3542	+static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
	3543	+{
	3544	+ /* Initial reset is a superset of the normal reset */
	3545	+ kvm_arch_vcpu_ioctl_normal_reset(vcpu);
	3546	+
	3547	+ /*
	3548	+ * This equals initial cpu reset in pop, but we don't switch to ESA.
	3549	+ * We do not only reset the internal data, but also ...
	3550	+ */
	3551	+ vcpu->arch.sie_block->gpsw.mask = 0;
	3552	+ vcpu->arch.sie_block->gpsw.addr = 0;
	3553	+ kvm_s390_set_prefix(vcpu, 0);
	3554	+ kvm_s390_set_cpu_timer(vcpu, 0);
	3555	+ vcpu->arch.sie_block->ckc = 0;
	3556	+ memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
	3557	+ vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
	3558	+ vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
	3559	+
	3560	+ /* ... the data in sync regs */
	3561	+ memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
	3562	+ vcpu->run->s.regs.ckc = 0;
	3563	+ vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
	3564	+ vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
	3565	+ vcpu->run->psw_addr = 0;
	3566	+ vcpu->run->psw_mask = 0;
	3567	+ vcpu->run->s.regs.todpr = 0;
	3568	+ vcpu->run->s.regs.cputm = 0;
	3569	+ vcpu->run->s.regs.ckc = 0;
	3570	+ vcpu->run->s.regs.pp = 0;
	3571	+ vcpu->run->s.regs.gbea = 1;
	3572	+ vcpu->run->s.regs.fpc = 0;
	3573	+ /*
	3574	+ * Do not reset these registers in the protected case, as some of
	3575	+ * them are overlayed and they are not accessible in this case
	3576	+ * anyway.
	3577	+ */
	3578	+ if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
	3579	+ vcpu->arch.sie_block->gbea = 1;
	3580	+ vcpu->arch.sie_block->pp = 0;
	3581	+ vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
	3582	+ vcpu->arch.sie_block->todpr = 0;
	3583	+ }
	3584	+}
	3585	+
	3586	+static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
	3587	+{
	3588	+ struct kvm_sync_regs *regs = &vcpu->run->s.regs;
	3589	+
	3590	+ /* Clear reset is a superset of the initial reset */
	3591	+ kvm_arch_vcpu_ioctl_initial_reset(vcpu);
	3592	+
	3593	+ memset(&regs->gprs, 0, sizeof(regs->gprs));
	3594	+ memset(&regs->vrs, 0, sizeof(regs->vrs));
	3595	+ memset(&regs->acrs, 0, sizeof(regs->acrs));
	3596	+ memset(&regs->gscb, 0, sizeof(regs->gscb));
	3597	+
	3598	+ regs->etoken = 0;
	3599	+ regs->etoken_extension = 0;
2946	3600	}
2947	3601
2948	3602	int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu vcpu, struct kvm_regs regs)
..	..	@@ -3116,14 +3770,20 @@
3116	3770
3117	3771	switch (mp_state->mp_state) {
3118	3772	case KVM_MP_STATE_STOPPED:
3119		- kvm_s390_vcpu_stop(vcpu);
	3773	+ rc = kvm_s390_vcpu_stop(vcpu);
3120	3774	break;
3121	3775	case KVM_MP_STATE_OPERATING:
3122		- kvm_s390_vcpu_start(vcpu);
	3776	+ rc = kvm_s390_vcpu_start(vcpu);
3123	3777	break;
3124	3778	case KVM_MP_STATE_LOAD:
	3779	+ if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
	3780	+ rc = -ENXIO;
	3781	+ break;
	3782	+ }
	3783	+ rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
	3784	+ break;
3125	3785	case KVM_MP_STATE_CHECK_STOP:
3126		- /* fall through - CHECK_STOP and LOAD are not supported yet */
	3786	+ fallthrough; /* CHECK_STOP and LOAD are not supported yet */
3127	3787	default:
3128	3788	rc = -ENXIO;
3129	3789	}
..	..	@@ -3211,18 +3871,18 @@
3211	3871
3212	3872	/* nothing to do, just clear the request */
3213	3873	kvm_clear_request(KVM_REQ_UNHALT, vcpu);
	3874	+ /* we left the vsie handler, nothing to do, just clear the request */
	3875	+ kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
3214	3876
3215	3877	return 0;
3216	3878	}
3217	3879
3218		-void kvm_s390_set_tod_clock(struct kvm *kvm,
3219		- const struct kvm_s390_vm_tod_clock *gtod)
	3880	+static void __kvm_s390_set_tod_clock(struct kvm kvm, const struct kvm_s390_vm_tod_clock gtod)
3220	3881	{
3221	3882	struct kvm_vcpu *vcpu;
3222	3883	struct kvm_s390_tod_clock_ext htod;
3223	3884	int i;
3224	3885
3225		- mutex_lock(&kvm->lock);
3226	3886	preempt_disable();
3227	3887
3228	3888	get_tod_clock_ext((char *)&htod);
..	..	@@ -3243,7 +3903,15 @@
3243	3903
3244	3904	kvm_s390_vcpu_unblock_all(kvm);
3245	3905	preempt_enable();
	3906	+}
	3907	+
	3908	+int kvm_s390_try_set_tod_clock(struct kvm kvm, const struct kvm_s390_vm_tod_clock gtod)
	3909	+{
	3910	+ if (!mutex_trylock(&kvm->lock))
	3911	+ return 0;
	3912	+ __kvm_s390_set_tod_clock(kvm, gtod);
3246	3913	mutex_unlock(&kvm->lock);
	3914	+ return 1;
3247	3915	}
3248	3916
3249	3917	/**
..	..	@@ -3279,11 +3947,13 @@
3279	3947	}
3280	3948	}
3281	3949
3282		-void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
	3950	+bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3283	3951	struct kvm_async_pf *work)
3284	3952	{
3285	3953	trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3286	3954	__kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
	3955	+
	3956	+ return true;
3287	3957	}
3288	3958
3289	3959	void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
..	..	@@ -3299,7 +3969,7 @@
3299	3969	/* s390 will always inject the page directly */
3300	3970	}
3301	3971
3302		-bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
	3972	+bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
3303	3973	{
3304	3974	/*
3305	3975	* s390 will always inject the page directly,
..	..	@@ -3308,33 +3978,31 @@
3308	3978	return true;
3309	3979	}
3310	3980
3311		-static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
	3981	+static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3312	3982	{
3313	3983	hva_t hva;
3314	3984	struct kvm_arch_async_pf arch;
3315		- int rc;
3316	3985
3317	3986	if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3318		- return 0;
	3987	+ return false;
3319	3988	if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3320	3989	vcpu->arch.pfault_compare)
3321		- return 0;
	3990	+ return false;
3322	3991	if (psw_extint_disabled(vcpu))
3323		- return 0;
	3992	+ return false;
3324	3993	if (kvm_s390_vcpu_has_irq(vcpu, 0))
3325		- return 0;
	3994	+ return false;
3326	3995	if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3327		- return 0;
	3996	+ return false;
3328	3997	if (!vcpu->arch.gmap->pfault_enabled)
3329		- return 0;
	3998	+ return false;
3330	3999
3331	4000	hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3332	4001	hva += current->thread.gmap_addr & ~PAGE_MASK;
3333	4002	if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3334		- return 0;
	4003	+ return false;
3335	4004
3336		- rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3337		- return rc;
	4005	+ return kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3338	4006	}
3339	4007
3340	4008	static int vcpu_pre_run(struct kvm_vcpu *vcpu)
..	..	@@ -3354,9 +4022,6 @@
3354	4022	if (need_resched())
3355	4023	schedule();
3356	4024
3357		- if (test_cpu_flag(CIF_MCCK_PENDING))
3358		- s390_handle_mcck();
3359		-
3360	4025	if (!kvm_is_ucontrol(vcpu->kvm)) {
3361	4026	rc = kvm_s390_deliver_pending_interrupts(vcpu);
3362	4027	if (rc)
..	..	@@ -3371,6 +4036,8 @@
3371	4036	kvm_s390_backup_guest_per_regs(vcpu);
3372	4037	kvm_s390_patch_guest_per_regs(vcpu);
3373	4038	}
	4039	+
	4040	+ clear_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.gisa_int.kicked_mask);
3374	4041
3375	4042	vcpu->arch.sie_block->icptcode = 0;
3376	4043	cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
..	..	@@ -3469,9 +4136,11 @@
3469	4136	return vcpu_post_run_fault_in_sie(vcpu);
3470	4137	}
3471	4138
	4139	+#define PSW_INT_MASK (PSW_MASK_EXT \| PSW_MASK_IO \| PSW_MASK_MCHECK)
3472	4140	static int __vcpu_run(struct kvm_vcpu *vcpu)
3473	4141	{
3474	4142	int rc, exit_reason;
	4143	+ struct sie_page sie_page = (struct sie_page )vcpu->arch.sie_block;
3475	4144
3476	4145	/*
3477	4146	* We try to hold kvm->srcu during most of vcpu_run (except when run-
..	..	@@ -3493,8 +4162,28 @@
3493	4162	guest_enter_irqoff();
3494	4163	__disable_cpu_timer_accounting(vcpu);
3495	4164	local_irq_enable();
	4165	+ if (kvm_s390_pv_cpu_is_protected(vcpu)) {
	4166	+ memcpy(sie_page->pv_grregs,
	4167	+ vcpu->run->s.regs.gprs,
	4168	+ sizeof(sie_page->pv_grregs));
	4169	+ }
3496	4170	exit_reason = sie64a(vcpu->arch.sie_block,
3497	4171	vcpu->run->s.regs.gprs);
	4172	+ if (kvm_s390_pv_cpu_is_protected(vcpu)) {
	4173	+ memcpy(vcpu->run->s.regs.gprs,
	4174	+ sie_page->pv_grregs,
	4175	+ sizeof(sie_page->pv_grregs));
	4176	+ /*
	4177	+ * We're not allowed to inject interrupts on intercepts
	4178	+ * that leave the guest state in an "in-between" state
	4179	+ * where the next SIE entry will do a continuation.
	4180	+ * Fence interrupts in our "internal" PSW.
	4181	+ */
	4182	+ if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR \|\|
	4183	+ vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
	4184	+ vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
	4185	+ }
	4186	+ }
3498	4187	local_irq_disable();
3499	4188	__enable_cpu_timer_accounting(vcpu);
3500	4189	guest_exit_irqoff();
..	..	@@ -3508,8 +4197,9 @@
3508	4197	return rc;
3509	4198	}
3510	4199
3511		-static void sync_regs(struct kvm_vcpu vcpu, struct kvm_run kvm_run)
	4200	+static void sync_regs_fmt2(struct kvm_vcpu *vcpu)
3512	4201	{
	4202	+ struct kvm_run *kvm_run = vcpu->run;
3513	4203	struct runtime_instr_cb *riccb;
3514	4204	struct gs_cb *gscb;
3515	4205
..	..	@@ -3517,16 +4207,7 @@
3517	4207	gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3518	4208	vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3519	4209	vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3520		- if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3521		- kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3522		- if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3523		- memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3524		- /* some control register changes require a tlb flush */
3525		- kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3526		- }
3527	4210	if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3528		- kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3529		- vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3530	4211	vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3531	4212	vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3532	4213	vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
..	..	@@ -3537,6 +4218,10 @@
3537	4218	vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3538	4219	if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3539	4220	kvm_clear_async_pf_completion_queue(vcpu);
	4221	+ }
	4222	+ if (kvm_run->kvm_dirty_regs & KVM_SYNC_DIAG318) {
	4223	+ vcpu->arch.diag318_info.val = kvm_run->s.regs.diag318;
	4224	+ vcpu->arch.sie_block->cpnc = vcpu->arch.diag318_info.cpnc;
3540	4225	}
3541	4226	/*
3542	4227	* If userspace sets the riccb (e.g. after migration) to a valid state,
..	..	@@ -3567,20 +4252,6 @@
3567	4252	vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3568	4253	vcpu->arch.sie_block->fpf \|= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3569	4254	}
3570		- save_access_regs(vcpu->arch.host_acrs);
3571		- restore_access_regs(vcpu->run->s.regs.acrs);
3572		- /* save host (userspace) fprs/vrs */
3573		- save_fpu_regs();
3574		- vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3575		- vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3576		- if (MACHINE_HAS_VX)
3577		- current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3578		- else
3579		- current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3580		- current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3581		- if (test_fp_ctl(current->thread.fpu.fpc))
3582		- /* User space provided an invalid FPC, let's clear it */
3583		- current->thread.fpu.fpc = 0;
3584	4255	if (MACHINE_HAS_GS) {
3585	4256	preempt_disable();
3586	4257	__ctl_set_bit(2, 4);
..	..	@@ -3596,33 +4267,68 @@
3596	4267	preempt_enable();
3597	4268	}
3598	4269	/* SIE will load etoken directly from SDNX and therefore kvm_run */
	4270	+}
	4271	+
	4272	+static void sync_regs(struct kvm_vcpu *vcpu)
	4273	+{
	4274	+ struct kvm_run *kvm_run = vcpu->run;
	4275	+
	4276	+ if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
	4277	+ kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
	4278	+ if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
	4279	+ memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
	4280	+ /* some control register changes require a tlb flush */
	4281	+ kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
	4282	+ }
	4283	+ if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
	4284	+ kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
	4285	+ vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
	4286	+ }
	4287	+ save_access_regs(vcpu->arch.host_acrs);
	4288	+ restore_access_regs(vcpu->run->s.regs.acrs);
	4289	+ /* save host (userspace) fprs/vrs */
	4290	+ save_fpu_regs();
	4291	+ vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
	4292	+ vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
	4293	+ if (MACHINE_HAS_VX)
	4294	+ current->thread.fpu.regs = vcpu->run->s.regs.vrs;
	4295	+ else
	4296	+ current->thread.fpu.regs = vcpu->run->s.regs.fprs;
	4297	+ current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
	4298	+ if (test_fp_ctl(current->thread.fpu.fpc))
	4299	+ /* User space provided an invalid FPC, let's clear it */
	4300	+ current->thread.fpu.fpc = 0;
	4301	+
	4302	+ /* Sync fmt2 only data */
	4303	+ if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
	4304	+ sync_regs_fmt2(vcpu);
	4305	+ } else {
	4306	+ /*
	4307	+ * In several places we have to modify our internal view to
	4308	+ * not do things that are disallowed by the ultravisor. For
	4309	+ * example we must not inject interrupts after specific exits
	4310	+ * (e.g. 112 prefix page not secure). We do this by turning
	4311	+ * off the machine check, external and I/O interrupt bits
	4312	+ * of our PSW copy. To avoid getting validity intercepts, we
	4313	+ * do only accept the condition code from userspace.
	4314	+ */
	4315	+ vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
	4316	+ vcpu->arch.sie_block->gpsw.mask \|= kvm_run->psw_mask &
	4317	+ PSW_MASK_CC;
	4318	+ }
3599	4319
3600	4320	kvm_run->kvm_dirty_regs = 0;
3601	4321	}
3602	4322
3603		-static void store_regs(struct kvm_vcpu vcpu, struct kvm_run kvm_run)
	4323	+static void store_regs_fmt2(struct kvm_vcpu *vcpu)
3604	4324	{
3605		- kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3606		- kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3607		- kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3608		- memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3609		- kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3610		- kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
	4325	+ struct kvm_run *kvm_run = vcpu->run;
	4326	+
3611	4327	kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3612	4328	kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3613	4329	kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3614		- kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3615		- kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3616		- kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3617	4330	kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
3618		- save_access_regs(vcpu->run->s.regs.acrs);
3619		- restore_access_regs(vcpu->arch.host_acrs);
3620		- /* Save guest register state */
3621		- save_fpu_regs();
3622		- vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3623		- /* Restore will be done lazily at return */
3624		- current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3625		- current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
	4331	+ kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val;
3626	4332	if (MACHINE_HAS_GS) {
3627	4333	preempt_disable();
3628	4334	__ctl_set_bit(2, 4);
..	..	@@ -3638,12 +4344,42 @@
3638	4344	/* SIE will save etoken directly into SDNX and therefore kvm_run */
3639	4345	}
3640	4346
3641		-int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run kvm_run)
	4347	+static void store_regs(struct kvm_vcpu *vcpu)
3642	4348	{
	4349	+ struct kvm_run *kvm_run = vcpu->run;
	4350	+
	4351	+ kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
	4352	+ kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
	4353	+ kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
	4354	+ memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
	4355	+ kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
	4356	+ kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
	4357	+ kvm_run->s.regs.pft = vcpu->arch.pfault_token;
	4358	+ kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
	4359	+ kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
	4360	+ save_access_regs(vcpu->run->s.regs.acrs);
	4361	+ restore_access_regs(vcpu->arch.host_acrs);
	4362	+ /* Save guest register state */
	4363	+ save_fpu_regs();
	4364	+ vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
	4365	+ /* Restore will be done lazily at return */
	4366	+ current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
	4367	+ current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
	4368	+ if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
	4369	+ store_regs_fmt2(vcpu);
	4370	+}
	4371	+
	4372	+int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
	4373	+{
	4374	+ struct kvm_run *kvm_run = vcpu->run;
3643	4375	int rc;
3644	4376
3645	4377	if (kvm_run->immediate_exit)
3646	4378	return -EINTR;
	4379	+
	4380	+ if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS \|\|
	4381	+ kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
	4382	+ return -EINVAL;
3647	4383
3648	4384	vcpu_load(vcpu);
3649	4385
..	..	@@ -3655,6 +4391,10 @@
3655	4391
3656	4392	kvm_sigset_activate(vcpu);
3657	4393
	4394	+ /*
	4395	+ * no need to check the return value of vcpu_start as it can only have
	4396	+ * an error for protvirt, but protvirt means user cpu state
	4397	+ */
3658	4398	if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3659	4399	kvm_s390_vcpu_start(vcpu);
3660	4400	} else if (is_vcpu_stopped(vcpu)) {
..	..	@@ -3664,7 +4404,7 @@
3664	4404	goto out;
3665	4405	}
3666	4406
3667		- sync_regs(vcpu, kvm_run);
	4407	+ sync_regs(vcpu);
3668	4408	enable_cpu_timer_accounting(vcpu);
3669	4409
3670	4410	might_fault();
..	..	@@ -3686,7 +4426,7 @@
3686	4426	}
3687	4427
3688	4428	disable_cpu_timer_accounting(vcpu);
3689		- store_regs(vcpu, kvm_run);
	4429	+ store_regs(vcpu);
3690	4430
3691	4431	kvm_sigset_deactivate(vcpu);
3692	4432
..	..	@@ -3792,17 +4532,26 @@
3792	4532	kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
3793	4533	}
3794	4534
3795		-void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
	4535	+int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3796	4536	{
3797		- int i, online_vcpus, started_vcpus = 0;
	4537	+ int i, online_vcpus, r = 0, started_vcpus = 0;
3798	4538
3799	4539	if (!is_vcpu_stopped(vcpu))
3800		- return;
	4540	+ return 0;
3801	4541
3802	4542	trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
3803	4543	/* Only one cpu at a time may enter/leave the STOPPED state. */
3804	4544	spin_lock(&vcpu->kvm->arch.start_stop_lock);
3805	4545	online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
	4546	+
	4547	+ /* Let's tell the UV that we want to change into the operating state */
	4548	+ if (kvm_s390_pv_cpu_is_protected(vcpu)) {
	4549	+ r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
	4550	+ if (r) {
	4551	+ spin_unlock(&vcpu->kvm->arch.start_stop_lock);
	4552	+ return r;
	4553	+ }
	4554	+ }
3806	4555
3807	4556	for (i = 0; i < online_vcpus; i++) {
3808	4557	if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
..	..	@@ -3823,26 +4572,42 @@
3823	4572
3824	4573	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
3825	4574	/*
	4575	+ * The real PSW might have changed due to a RESTART interpreted by the
	4576	+ * ultravisor. We block all interrupts and let the next sie exit
	4577	+ * refresh our view.
	4578	+ */
	4579	+ if (kvm_s390_pv_cpu_is_protected(vcpu))
	4580	+ vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
	4581	+ /*
3826	4582	* Another VCPU might have used IBS while we were offline.
3827	4583	* Let's play safe and flush the VCPU at startup.
3828	4584	*/
3829	4585	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3830	4586	spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3831		- return;
	4587	+ return 0;
3832	4588	}
3833	4589
3834		-void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
	4590	+int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3835	4591	{
3836		- int i, online_vcpus, started_vcpus = 0;
	4592	+ int i, online_vcpus, r = 0, started_vcpus = 0;
3837	4593	struct kvm_vcpu *started_vcpu = NULL;
3838	4594
3839	4595	if (is_vcpu_stopped(vcpu))
3840		- return;
	4596	+ return 0;
3841	4597
3842	4598	trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
3843	4599	/* Only one cpu at a time may enter/leave the STOPPED state. */
3844	4600	spin_lock(&vcpu->kvm->arch.start_stop_lock);
3845	4601	online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
	4602	+
	4603	+ /* Let's tell the UV that we want to change into the stopped state */
	4604	+ if (kvm_s390_pv_cpu_is_protected(vcpu)) {
	4605	+ r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
	4606	+ if (r) {
	4607	+ spin_unlock(&vcpu->kvm->arch.start_stop_lock);
	4608	+ return r;
	4609	+ }
	4610	+ }
3846	4611
3847	4612	/*
3848	4613	* Set the VCPU to STOPPED and THEN clear the interrupt flag,
..	..	@@ -3871,7 +4636,7 @@
3871	4636	}
3872	4637
3873	4638	spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3874		- return;
	4639	+ return 0;
3875	4640	}
3876	4641
3877	4642	static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
..	..	@@ -3898,12 +4663,42 @@
3898	4663	return r;
3899	4664	}
3900	4665
	4666	+static long kvm_s390_guest_sida_op(struct kvm_vcpu *vcpu,
	4667	+ struct kvm_s390_mem_op *mop)
	4668	+{
	4669	+ void __user uaddr = (void __user )mop->buf;
	4670	+ int r = 0;
	4671	+
	4672	+ if (mop->flags \|\| !mop->size)
	4673	+ return -EINVAL;
	4674	+ if (mop->size + mop->sida_offset < mop->size)
	4675	+ return -EINVAL;
	4676	+ if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
	4677	+ return -E2BIG;
	4678	+ if (!kvm_s390_pv_cpu_is_protected(vcpu))
	4679	+ return -EINVAL;
	4680	+
	4681	+ switch (mop->op) {
	4682	+ case KVM_S390_MEMOP_SIDA_READ:
	4683	+ if (copy_to_user(uaddr, (void *)(sida_origin(vcpu->arch.sie_block) +
	4684	+ mop->sida_offset), mop->size))
	4685	+ r = -EFAULT;
	4686	+
	4687	+ break;
	4688	+ case KVM_S390_MEMOP_SIDA_WRITE:
	4689	+ if (copy_from_user((void *)(sida_origin(vcpu->arch.sie_block) +
	4690	+ mop->sida_offset), uaddr, mop->size))
	4691	+ r = -EFAULT;
	4692	+ break;
	4693	+ }
	4694	+ return r;
	4695	+}
3901	4696	static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3902	4697	struct kvm_s390_mem_op *mop)
3903	4698	{
3904	4699	void __user uaddr = (void __user )mop->buf;
3905	4700	void *tmpbuf = NULL;
3906		- int r, srcu_idx;
	4701	+ int r = 0;
3907	4702	const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3908	4703	\| KVM_S390_MEMOP_F_CHECK_ONLY;
3909	4704
..	..	@@ -3913,13 +4708,14 @@
3913	4708	if (mop->size > MEM_OP_MAX_SIZE)
3914	4709	return -E2BIG;
3915	4710
	4711	+ if (kvm_s390_pv_cpu_is_protected(vcpu))
	4712	+ return -EINVAL;
	4713	+
3916	4714	if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3917	4715	tmpbuf = vmalloc(mop->size);
3918	4716	if (!tmpbuf)
3919	4717	return -ENOMEM;
3920	4718	}
3921		-
3922		- srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3923	4719
3924	4720	switch (mop->op) {
3925	4721	case KVM_S390_MEMOP_LOGICAL_READ:
..	..	@@ -3946,16 +4742,37 @@
3946	4742	}
3947	4743	r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3948	4744	break;
3949		- default:
3950		- r = -EINVAL;
3951	4745	}
3952		-
3953		- srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3954	4746
3955	4747	if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3956	4748	kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3957	4749
3958	4750	vfree(tmpbuf);
	4751	+ return r;
	4752	+}
	4753	+
	4754	+static long kvm_s390_guest_memsida_op(struct kvm_vcpu *vcpu,
	4755	+ struct kvm_s390_mem_op *mop)
	4756	+{
	4757	+ int r, srcu_idx;
	4758	+
	4759	+ srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
	4760	+
	4761	+ switch (mop->op) {
	4762	+ case KVM_S390_MEMOP_LOGICAL_READ:
	4763	+ case KVM_S390_MEMOP_LOGICAL_WRITE:
	4764	+ r = kvm_s390_guest_mem_op(vcpu, mop);
	4765	+ break;
	4766	+ case KVM_S390_MEMOP_SIDA_READ:
	4767	+ case KVM_S390_MEMOP_SIDA_WRITE:
	4768	+ /* we are locked against sida going away by the vcpu->mutex */
	4769	+ r = kvm_s390_guest_sida_op(vcpu, mop);
	4770	+ break;
	4771	+ default:
	4772	+ r = -EINVAL;
	4773	+ }
	4774	+
	4775	+ srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3959	4776	return r;
3960	4777	}
3961	4778
..	..	@@ -3994,6 +4811,7 @@
3994	4811	void __user argp = (void __user )arg;
3995	4812	int idx;
3996	4813	long r;
	4814	+ u16 rc, rrc;
3997	4815
3998	4816	vcpu_load(vcpu);
3999	4817
..	..	@@ -4012,12 +4830,43 @@
4012	4830	r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4013	4831	break;
4014	4832	}
	4833	+ case KVM_S390_CLEAR_RESET:
	4834	+ r = 0;
	4835	+ kvm_arch_vcpu_ioctl_clear_reset(vcpu);
	4836	+ if (kvm_s390_pv_cpu_is_protected(vcpu)) {
	4837	+ r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
	4838	+ UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
	4839	+ VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
	4840	+ rc, rrc);
	4841	+ }
	4842	+ break;
4015	4843	case KVM_S390_INITIAL_RESET:
4016		- r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
	4844	+ r = 0;
	4845	+ kvm_arch_vcpu_ioctl_initial_reset(vcpu);
	4846	+ if (kvm_s390_pv_cpu_is_protected(vcpu)) {
	4847	+ r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
	4848	+ UVC_CMD_CPU_RESET_INITIAL,
	4849	+ &rc, &rrc);
	4850	+ VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
	4851	+ rc, rrc);
	4852	+ }
	4853	+ break;
	4854	+ case KVM_S390_NORMAL_RESET:
	4855	+ r = 0;
	4856	+ kvm_arch_vcpu_ioctl_normal_reset(vcpu);
	4857	+ if (kvm_s390_pv_cpu_is_protected(vcpu)) {
	4858	+ r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
	4859	+ UVC_CMD_CPU_RESET, &rc, &rrc);
	4860	+ VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
	4861	+ rc, rrc);
	4862	+ }
4017	4863	break;
4018	4864	case KVM_SET_ONE_REG:
4019	4865	case KVM_GET_ONE_REG: {
4020	4866	struct kvm_one_reg reg;
	4867	+ r = -EINVAL;
	4868	+ if (kvm_s390_pv_cpu_is_protected(vcpu))
	4869	+ break;
4021	4870	r = -EFAULT;
4022	4871	if (copy_from_user(&reg, argp, sizeof(reg)))
4023	4872	break;
..	..	@@ -4080,7 +4929,7 @@
4080	4929	struct kvm_s390_mem_op mem_op;
4081	4930
4082	4931	if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4083		- r = kvm_s390_guest_mem_op(vcpu, &mem_op);
	4932	+ r = kvm_s390_guest_memsida_op(vcpu, &mem_op);
4084	4933	else
4085	4934	r = -EFAULT;
4086	4935	break;
..	..	@@ -4140,12 +4989,6 @@
4140	4989	return VM_FAULT_SIGBUS;
4141	4990	}
4142	4991
4143		-int kvm_arch_create_memslot(struct kvm kvm, struct kvm_memory_slot slot,
4144		- unsigned long npages)
4145		-{
4146		- return 0;
4147		-}
4148		-
4149	4992	/* Section: memory related */
4150	4993	int kvm_arch_prepare_memory_region(struct kvm *kvm,
4151	4994	struct kvm_memory_slot *memslot,
..	..	@@ -4166,12 +5009,15 @@
4166	5009	if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4167	5010	return -EINVAL;
4168	5011
	5012	+ /* When we are protected, we should not change the memory slots */
	5013	+ if (kvm_s390_pv_get_handle(kvm))
	5014	+ return -EINVAL;
4169	5015	return 0;
4170	5016	}
4171	5017
4172	5018	void kvm_arch_commit_memory_region(struct kvm *kvm,
4173	5019	const struct kvm_userspace_memory_region *mem,
4174		- const struct kvm_memory_slot *old,
	5020	+ struct kvm_memory_slot *old,
4175	5021	const struct kvm_memory_slot *new,
4176	5022	enum kvm_mr_change change)
4177	5023	{
..	..	@@ -4187,7 +5033,7 @@
4187	5033	old->npages * PAGE_SIZE);
4188	5034	if (rc)
4189	5035	break;
4190		- /* FALLTHROUGH */
	5036	+ fallthrough;
4191	5037	case KVM_MR_CREATE:
4192	5038	rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4193	5039	mem->guest_phys_addr, mem->memory_size);
..	..	@@ -4219,12 +5065,12 @@
4219	5065	int i;
4220	5066
4221	5067	if (!sclp.has_sief2) {
4222		- pr_info("SIE not available\n");
	5068	+ pr_info("SIE is not available\n");
4223	5069	return -ENODEV;
4224	5070	}
4225	5071
4226	5072	if (nested && hpage) {
4227		- pr_info("nested (vSIE) and hpage (huge page backing) can currently not be activated concurrently");
	5073	+ pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
4228	5074	return -EINVAL;
4229	5075	}
4230	5076