forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 1f93a7dfd1f8d5ff7a5c53246c7534fe2332d6f4 
 kernel/arch/s390/kvm/vsie.c
....@@ -135,13 +135,147 @@
135135 	atomic_set(&scb_s->cpuflags, newflags);
136136 	return 0;
137137 }
138
+/* Copy to APCB FORMAT1 from APCB FORMAT0 */
139
+static int setup_apcb10(struct kvm_vcpu *vcpu, struct kvm_s390_apcb1 *apcb_s,
140
+			unsigned long apcb_o, struct kvm_s390_apcb1 *apcb_h)
141
+{
142
+	struct kvm_s390_apcb0 tmp;
138143 
139
-/*
144
+	if (read_guest_real(vcpu, apcb_o, &tmp, sizeof(struct kvm_s390_apcb0)))
145
+		return -EFAULT;
146
+
147
+	apcb_s->apm[0] = apcb_h->apm[0] & tmp.apm[0];
148
+	apcb_s->aqm[0] = apcb_h->aqm[0] & tmp.aqm[0] & 0xffff000000000000UL;
149
+	apcb_s->adm[0] = apcb_h->adm[0] & tmp.adm[0] & 0xffff000000000000UL;
150
+
151
+	return 0;
152
+
153
+}
154
+
155
+/**
156
+ * setup_apcb00 - Copy to APCB FORMAT0 from APCB FORMAT0
157
+ * @vcpu: pointer to the virtual CPU
158
+ * @apcb_s: pointer to start of apcb in the shadow crycb
159
+ * @apcb_o: pointer to start of original apcb in the guest2
160
+ * @apcb_h: pointer to start of apcb in the guest1
161
+ *
162
+ * Returns 0 and -EFAULT on error reading guest apcb
163
+ */
164
+static int setup_apcb00(struct kvm_vcpu *vcpu, unsigned long *apcb_s,
165
+			unsigned long apcb_o, unsigned long *apcb_h)
166
+{
167
+	if (read_guest_real(vcpu, apcb_o, apcb_s,
168
+			    sizeof(struct kvm_s390_apcb0)))
169
+		return -EFAULT;
170
+
171
+	bitmap_and(apcb_s, apcb_s, apcb_h, sizeof(struct kvm_s390_apcb0));
172
+
173
+	return 0;
174
+}
175
+
176
+/**
177
+ * setup_apcb11 - Copy the FORMAT1 APCB from the guest to the shadow CRYCB
178
+ * @vcpu: pointer to the virtual CPU
179
+ * @apcb_s: pointer to start of apcb in the shadow crycb
180
+ * @apcb_o: pointer to start of original guest apcb
181
+ * @apcb_h: pointer to start of apcb in the host
182
+ *
183
+ * Returns 0 and -EFAULT on error reading guest apcb
184
+ */
185
+static int setup_apcb11(struct kvm_vcpu *vcpu, unsigned long *apcb_s,
186
+			unsigned long apcb_o,
187
+			unsigned long *apcb_h)
188
+{
189
+	if (read_guest_real(vcpu, apcb_o, apcb_s,
190
+			    sizeof(struct kvm_s390_apcb1)))
191
+		return -EFAULT;
192
+
193
+	bitmap_and(apcb_s, apcb_s, apcb_h, sizeof(struct kvm_s390_apcb1));
194
+
195
+	return 0;
196
+}
197
+
198
+/**
199
+ * setup_apcb - Create a shadow copy of the apcb.
200
+ * @vcpu: pointer to the virtual CPU
201
+ * @crycb_s: pointer to shadow crycb
202
+ * @crycb_o: pointer to original guest crycb
203
+ * @crycb_h: pointer to the host crycb
204
+ * @fmt_o: format of the original guest crycb.
205
+ * @fmt_h: format of the host crycb.
206
+ *
207
+ * Checks the compatibility between the guest and host crycb and calls the
208
+ * appropriate copy function.
209
+ *
210
+ * Return 0 or an error number if the guest and host crycb are incompatible.
211
+ */
212
+static int setup_apcb(struct kvm_vcpu *vcpu, struct kvm_s390_crypto_cb *crycb_s,
213
+	       const u32 crycb_o,
214
+	       struct kvm_s390_crypto_cb *crycb_h,
215
+	       int fmt_o, int fmt_h)
216
+{
217
+	struct kvm_s390_crypto_cb *crycb;
218
+
219
+	crycb = (struct kvm_s390_crypto_cb *) (unsigned long)crycb_o;
220
+
221
+	switch (fmt_o) {
222
+	case CRYCB_FORMAT2:
223
+		if ((crycb_o & PAGE_MASK) != ((crycb_o + 256) & PAGE_MASK))
224
+			return -EACCES;
225
+		if (fmt_h != CRYCB_FORMAT2)
226
+			return -EINVAL;
227
+		return setup_apcb11(vcpu, (unsigned long *)&crycb_s->apcb1,
228
+				    (unsigned long) &crycb->apcb1,
229
+				    (unsigned long *)&crycb_h->apcb1);
230
+	case CRYCB_FORMAT1:
231
+		switch (fmt_h) {
232
+		case CRYCB_FORMAT2:
233
+			return setup_apcb10(vcpu, &crycb_s->apcb1,
234
+					    (unsigned long) &crycb->apcb0,
235
+					    &crycb_h->apcb1);
236
+		case CRYCB_FORMAT1:
237
+			return setup_apcb00(vcpu,
238
+					    (unsigned long *) &crycb_s->apcb0,
239
+					    (unsigned long) &crycb->apcb0,
240
+					    (unsigned long *) &crycb_h->apcb0);
241
+		}
242
+		break;
243
+	case CRYCB_FORMAT0:
244
+		if ((crycb_o & PAGE_MASK) != ((crycb_o + 32) & PAGE_MASK))
245
+			return -EACCES;
246
+
247
+		switch (fmt_h) {
248
+		case CRYCB_FORMAT2:
249
+			return setup_apcb10(vcpu, &crycb_s->apcb1,
250
+					    (unsigned long) &crycb->apcb0,
251
+					    &crycb_h->apcb1);
252
+		case CRYCB_FORMAT1:
253
+		case CRYCB_FORMAT0:
254
+			return setup_apcb00(vcpu,
255
+					    (unsigned long *) &crycb_s->apcb0,
256
+					    (unsigned long) &crycb->apcb0,
257
+					    (unsigned long *) &crycb_h->apcb0);
258
+		}
259
+	}
260
+	return -EINVAL;
261
+}
262
+
263
+/**
264
+ * shadow_crycb - Create a shadow copy of the crycb block
265
+ * @vcpu: a pointer to the virtual CPU
266
+ * @vsie_page: a pointer to internal date used for the vSIE
267
+ *
140268  * Create a shadow copy of the crycb block and setup key wrapping, if
141269  * requested for guest 3 and enabled for guest 2.
142270  *
143
- * We only accept format-1 (no AP in g2), but convert it into format-2
271
+ * We accept format-1 or format-2, but we convert format-1 into format-2
272
+ * in the shadow CRYCB.
273
+ * Using format-2 enables the firmware to choose the right format when
274
+ * scheduling the SIE.
144275  * There is nothing to do for format-0.
276
+ *
277
+ * This function centralize the issuing of set_validity_icpt() for all
278
+ * the subfunctions working on the crycb.
145279  *
146280  * Returns: - 0 if shadowed or nothing to do
147281  *          - > 0 if control has to be given to guest 2
....@@ -154,23 +288,44 @@
154288 	const u32 crycb_addr = crycbd_o & 0x7ffffff8U;
155289 	unsigned long *b1, *b2;
156290 	u8 ecb3_flags;
291
+	u32 ecd_flags;
292
+	int apie_h;
293
+	int apie_s;
294
+	int key_msk = test_kvm_facility(vcpu->kvm, 76);
295
+	int fmt_o = crycbd_o & CRYCB_FORMAT_MASK;
296
+	int fmt_h = vcpu->arch.sie_block->crycbd & CRYCB_FORMAT_MASK;
297
+	int ret = 0;
157298 
158299 	scb_s->crycbd = 0;
159
-	if (!(crycbd_o & vcpu->arch.sie_block->crycbd & CRYCB_FORMAT1))
300
+
301
+	apie_h = vcpu->arch.sie_block->eca & ECA_APIE;
302
+	apie_s = apie_h & scb_o->eca;
303
+	if (!apie_s && (!key_msk || (fmt_o == CRYCB_FORMAT0)))
160304 		return 0;
161
-	/* format-1 is supported with message-security-assist extension 3 */
162
-	if (!test_kvm_facility(vcpu->kvm, 76))
163
-		return 0;
305
+
306
+	if (!crycb_addr)
307
+		return set_validity_icpt(scb_s, 0x0039U);
308
+
309
+	if (fmt_o == CRYCB_FORMAT1)
310
+		if ((crycb_addr & PAGE_MASK) !=
311
+		    ((crycb_addr + 128) & PAGE_MASK))
312
+			return set_validity_icpt(scb_s, 0x003CU);
313
+
314
+	if (apie_s) {
315
+		ret = setup_apcb(vcpu, &vsie_page->crycb, crycb_addr,
316
+				 vcpu->kvm->arch.crypto.crycb,
317
+				 fmt_o, fmt_h);
318
+		if (ret)
319
+			goto end;
320
+		scb_s->eca |= scb_o->eca & ECA_APIE;
321
+	}
322
+
164323 	/* we may only allow it if enabled for guest 2 */
165324 	ecb3_flags = scb_o->ecb3 & vcpu->arch.sie_block->ecb3 &
166325 		     (ECB3_AES | ECB3_DEA);
167
-	if (!ecb3_flags)
168
-		return 0;
169
-
170
-	if ((crycb_addr & PAGE_MASK) != ((crycb_addr + 128) & PAGE_MASK))
171
-		return set_validity_icpt(scb_s, 0x003CU);
172
-	else if (!crycb_addr)
173
-		return set_validity_icpt(scb_s, 0x0039U);
326
+	ecd_flags = scb_o->ecd & vcpu->arch.sie_block->ecd & ECD_ECC;
327
+	if (!ecb3_flags && !ecd_flags)
328
+		goto end;
174329 
175330 	/* copy only the wrapping keys */
176331 	if (read_guest_real(vcpu, crycb_addr + 72,
....@@ -178,8 +333,7 @@
178333 		return set_validity_icpt(scb_s, 0x0035U);
179334 
180335 	scb_s->ecb3 |= ecb3_flags;
181
-	scb_s->crycbd = ((__u32)(__u64) &vsie_page->crycb) | CRYCB_FORMAT1 |
182
-			CRYCB_FORMAT2;
336
+	scb_s->ecd |= ecd_flags;
183337 
184338 	/* xor both blocks in one run */
185339 	b1 = (unsigned long *) vsie_page->crycb.dea_wrapping_key_mask;
....@@ -187,6 +341,16 @@
187341 			    vcpu->kvm->arch.crypto.crycb->dea_wrapping_key_mask;
188342 	/* as 56%8 == 0, bitmap_xor won't overwrite any data */
189343 	bitmap_xor(b1, b1, b2, BITS_PER_BYTE * 56);
344
+end:
345
+	switch (ret) {
346
+	case -EINVAL:
347
+		return set_validity_icpt(scb_s, 0x0022U);
348
+	case -EFAULT:
349
+		return set_validity_icpt(scb_s, 0x0035U);
350
+	case -EACCES:
351
+		return set_validity_icpt(scb_s, 0x003CU);
352
+	}
353
+	scb_s->crycbd = ((__u32)(__u64) &vsie_page->crycb) | CRYCB_FORMAT2;
190354 	return 0;
191355 }
192356 
....@@ -251,11 +415,6 @@
251415 	case ICPT_EXTINT:
252416 		memcpy((void *)((u64)scb_o + 0xc0),
253417 		       (void *)((u64)scb_s + 0xc0), 0xf0 - 0xc0);
254
-		break;
255
-	case ICPT_PARTEXEC:
256
-		/* MVPG only */
257
-		memcpy((void *)((u64)scb_o + 0xc0),
258
-		       (void *)((u64)scb_s + 0xc0), 0xd0 - 0xc0);
259418 		break;
260419 	}
261420 
....@@ -376,12 +535,17 @@
376535 	if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_CEI))
377536 		scb_s->eca |= scb_o->eca & ECA_CEI;
378537 	/* Epoch Extension */
379
-	if (test_kvm_facility(vcpu->kvm, 139))
538
+	if (test_kvm_facility(vcpu->kvm, 139)) {
380539 		scb_s->ecd |= scb_o->ecd & ECD_MEF;
540
+		scb_s->epdx = scb_o->epdx;
541
+	}
381542 
382543 	/* etoken */
383544 	if (test_kvm_facility(vcpu->kvm, 156))
384545 		scb_s->ecd |= scb_o->ecd & ECD_ETOKENF;
546
+
547
+	scb_s->hpid = HPID_VSIE;
548
+	scb_s->cpnc = scb_o->cpnc;
385549 
386550 	prepare_ibc(vcpu, vsie_page);
387551 	rc = shadow_crycb(vcpu, vsie_page);
....@@ -452,10 +616,10 @@
452616 	/* with mso/msl, the prefix lies at offset *mso* */
453617 	prefix += scb_s->mso;
454618 
455
-	rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, prefix);
619
+	rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, prefix, NULL);
456620 	if (!rc && (scb_s->ecb & ECB_TE))
457621 		rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap,
458
-					   prefix + PAGE_SIZE);
622
+					   prefix + PAGE_SIZE, NULL);
459623 	/*
460624 	 * We don't have to mprotect, we will be called for all unshadows.
461625 	 * SIE will detect if protection applies and trigger a validity.
....@@ -746,7 +910,7 @@
746910 				    current->thread.gmap_addr, 1);
747911 
748912 	rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap,
749
-				   current->thread.gmap_addr);
913
+				   current->thread.gmap_addr, NULL);
750914 	if (rc > 0) {
751915 		rc = inject_fault(vcpu, rc,
752916 				  current->thread.gmap_addr,
....@@ -768,7 +932,7 @@
768932 {
769933 	if (vsie_page->fault_addr)
770934 		kvm_s390_shadow_fault(vcpu, vsie_page->gmap,
771
-				      vsie_page->fault_addr);
935
+				      vsie_page->fault_addr, NULL);
772936 	vsie_page->fault_addr = 0;
773937 }
774938 
....@@ -816,6 +980,98 @@
816980 }
817981 
818982 /*
983
+ * Get a register for a nested guest.
984
+ * @vcpu the vcpu of the guest
985
+ * @vsie_page the vsie_page for the nested guest
986
+ * @reg the register number, the upper 4 bits are ignored.
987
+ * returns: the value of the register.
988
+ */
989
+static u64 vsie_get_register(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, u8 reg)
990
+{
991
+	/* no need to validate the parameter and/or perform error handling */
992
+	reg &= 0xf;
993
+	switch (reg) {
994
+	case 15:
995
+		return vsie_page->scb_s.gg15;
996
+	case 14:
997
+		return vsie_page->scb_s.gg14;
998
+	default:
999
+		return vcpu->run->s.regs.gprs[reg];
1000
+	}
1001
+}
1002
+
1003
+static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
1004
+{
1005
+	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
1006
+	unsigned long pei_dest, pei_src, src, dest, mask, prefix;
1007
+	u64 *pei_block = &vsie_page->scb_o->mcic;
1008
+	int edat, rc_dest, rc_src;
1009
+	union ctlreg0 cr0;
1010
+
1011
+	cr0.val = vcpu->arch.sie_block->gcr[0];
1012
+	edat = cr0.edat && test_kvm_facility(vcpu->kvm, 8);
1013
+	mask = _kvm_s390_logical_to_effective(&scb_s->gpsw, PAGE_MASK);
1014
+	prefix = scb_s->prefix << GUEST_PREFIX_SHIFT;
1015
+
1016
+	dest = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 20) & mask;
1017
+	dest = _kvm_s390_real_to_abs(prefix, dest) + scb_s->mso;
1018
+	src = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 16) & mask;
1019
+	src = _kvm_s390_real_to_abs(prefix, src) + scb_s->mso;
1020
+
1021
+	rc_dest = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, dest, &pei_dest);
1022
+	rc_src = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, src, &pei_src);
1023
+	/*
1024
+	 * Either everything went well, or something non-critical went wrong
1025
+	 * e.g. because of a race. In either case, simply retry.
1026
+	 */
1027
+	if (rc_dest == -EAGAIN || rc_src == -EAGAIN || (!rc_dest && !rc_src)) {
1028
+		retry_vsie_icpt(vsie_page);
1029
+		return -EAGAIN;
1030
+	}
1031
+	/* Something more serious went wrong, propagate the error */
1032
+	if (rc_dest < 0)
1033
+		return rc_dest;
1034
+	if (rc_src < 0)
1035
+		return rc_src;
1036
+
1037
+	/* The only possible suppressing exception: just deliver it */
1038
+	if (rc_dest == PGM_TRANSLATION_SPEC || rc_src == PGM_TRANSLATION_SPEC) {
1039
+		clear_vsie_icpt(vsie_page);
1040
+		rc_dest = kvm_s390_inject_program_int(vcpu, PGM_TRANSLATION_SPEC);
1041
+		WARN_ON_ONCE(rc_dest);
1042
+		return 1;
1043
+	}
1044
+
1045
+	/*
1046
+	 * Forward the PEI intercept to the guest if it was a page fault, or
1047
+	 * also for segment and region table faults if EDAT applies.
1048
+	 */
1049
+	if (edat) {
1050
+		rc_dest = rc_dest == PGM_ASCE_TYPE ? rc_dest : 0;
1051
+		rc_src = rc_src == PGM_ASCE_TYPE ? rc_src : 0;
1052
+	} else {
1053
+		rc_dest = rc_dest != PGM_PAGE_TRANSLATION ? rc_dest : 0;
1054
+		rc_src = rc_src != PGM_PAGE_TRANSLATION ? rc_src : 0;
1055
+	}
1056
+	if (!rc_dest && !rc_src) {
1057
+		pei_block[0] = pei_dest;
1058
+		pei_block[1] = pei_src;
1059
+		return 1;
1060
+	}
1061
+
1062
+	retry_vsie_icpt(vsie_page);
1063
+
1064
+	/*
1065
+	 * The host has edat, and the guest does not, or it was an ASCE type
1066
+	 * exception. The host needs to inject the appropriate DAT interrupts
1067
+	 * into the guest.
1068
+	 */
1069
+	if (rc_dest)
1070
+		return inject_fault(vcpu, rc_dest, dest, 1);
1071
+	return inject_fault(vcpu, rc_src, src, 0);
1072
+}
1073
+
1074
+/*
8191075  * Run the vsie on a shadow scb and a shadow gmap, without any further
8201076  * sanity checks, handling SIE faults.
8211077  *
....@@ -830,14 +1086,9 @@
8301086 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
8311087 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
8321088 	int guest_bp_isolation;
833
-	int rc;
1089
+	int rc = 0;
8341090 
8351091 	handle_last_fault(vcpu, vsie_page);
836
-
837
-	if (need_resched())
838
-		schedule();
839
-	if (test_cpu_flag(CIF_MCCK_PENDING))
840
-		s390_handle_mcck();
8411092 
8421093 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
8431094 
....@@ -858,7 +1109,18 @@
8581109 	guest_enter_irqoff();
8591110 	local_irq_enable();
8601111 
861
-	rc = sie64a(scb_s, vcpu->run->s.regs.gprs);
1112
+	/*
1113
+	 * Simulate a SIE entry of the VCPU (see sie64a), so VCPU blocking
1114
+	 * and VCPU requests also hinder the vSIE from running and lead
1115
+	 * to an immediate exit. kvm_s390_vsie_kick() has to be used to
1116
+	 * also kick the vSIE.
1117
+	 */
1118
+	vcpu->arch.sie_block->prog0c |= PROG_IN_SIE;
1119
+	barrier();
1120
+	if (!kvm_s390_vcpu_sie_inhibited(vcpu))
1121
+		rc = sie64a(scb_s, vcpu->run->s.regs.gprs);
1122
+	barrier();
1123
+	vcpu->arch.sie_block->prog0c &= ~PROG_IN_SIE;
8621124 
8631125 	local_irq_disable();
8641126 	guest_exit_irqoff();
....@@ -894,6 +1156,10 @@
8941156 	case ICPT_VALIDITY:
8951157 		if ((scb_s->ipa & 0xf000) != 0xf000)
8961158 			scb_s->ipa += 0x1000;
1159
+		break;
1160
+	case ICPT_PARTEXEC:
1161
+		if (scb_s->ipa == 0xb254)
1162
+			rc = vsie_handle_mvpg(vcpu, vsie_page);
8971163 		break;
8981164 	}
8991165 	return rc;
....@@ -1005,8 +1271,10 @@
10051271 		if (rc == -EAGAIN)
10061272 			rc = 0;
10071273 		if (rc || scb_s->icptcode || signal_pending(current) ||
1008
-		    kvm_s390_vcpu_has_irq(vcpu, 0))
1274
+		    kvm_s390_vcpu_has_irq(vcpu, 0) ||
1275
+		    kvm_s390_vcpu_sie_inhibited(vcpu))
10091276 			break;
1277
+		cond_resched();
10101278 	}
10111279 
10121280 	if (rc == -EFAULT) {
....@@ -1123,7 +1391,8 @@
11231391 	if (unlikely(scb_addr & 0x1ffUL))
11241392 		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
11251393 
1126
-	if (signal_pending(current) || kvm_s390_vcpu_has_irq(vcpu, 0))
1394
+	if (signal_pending(current) || kvm_s390_vcpu_has_irq(vcpu, 0) ||
1395
+	    kvm_s390_vcpu_sie_inhibited(vcpu))
11271396 		return 0;
11281397 
11291398 	vsie_page = get_vsie_page(vcpu->kvm, scb_addr);