forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 748e4f3d702def1a4bff191e0cf93b6a05340f01 
 kernel/arch/x86/kvm/ioapic.c
....@@ -46,13 +46,13 @@
4646 #include "lapic.h"
4747 #include "irq.h"
4848 
49
-#if 0
50
-#define ioapic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg)
51
-#else
52
-#define ioapic_debug(fmt, arg...)
53
-#endif
5449 static int ioapic_service(struct kvm_ioapic *vioapic, int irq,
5550 		bool line_status);
51
+
52
+static void kvm_ioapic_update_eoi_one(struct kvm_vcpu *vcpu,
53
+				      struct kvm_ioapic *ioapic,
54
+				      int trigger_mode,
55
+				      int pin);
5656 
5757 static unsigned long ioapic_read_indirect(struct kvm_ioapic *ioapic,
5858 					  unsigned long addr,
....@@ -115,8 +115,9 @@
115115 	union kvm_ioapic_redirect_entry *e;
116116 
117117 	e = &ioapic->redirtbl[RTC_GSI];
118
-	if (!kvm_apic_match_dest(vcpu, NULL, 0,	e->fields.dest_id,
119
-				e->fields.dest_mode))
118
+	if (!kvm_apic_match_dest(vcpu, NULL, APIC_DEST_NOSHORT,
119
+				 e->fields.dest_id,
120
+				 kvm_lapic_irq_dest_mode(!!e->fields.dest_mode)))
120121 		return;
121122 
122123 	new_val = kvm_apic_pending_eoi(vcpu, e->fields.vector);
....@@ -158,10 +159,16 @@
158159 	    __rtc_irq_eoi_tracking_restore_one(vcpu);
159160 }
160161 
161
-static void rtc_irq_eoi(struct kvm_ioapic *ioapic, struct kvm_vcpu *vcpu)
162
+static void rtc_irq_eoi(struct kvm_ioapic *ioapic, struct kvm_vcpu *vcpu,
163
+			int vector)
162164 {
163
-	if (test_and_clear_bit(vcpu->vcpu_id,
164
-			       ioapic->rtc_status.dest_map.map)) {
165
+	struct dest_map *dest_map = &ioapic->rtc_status.dest_map;
166
+
167
+	/* RTC special handling */
168
+	if (test_bit(vcpu->vcpu_id, dest_map->map) &&
169
+	    (vector == dest_map->vectors[vcpu->vcpu_id]) &&
170
+	    (test_and_clear_bit(vcpu->vcpu_id,
171
+				ioapic->rtc_status.dest_map.map))) {
165172 		--ioapic->rtc_status.pending_eoi;
166173 		rtc_status_pending_eoi_check_valid(ioapic);
167174 	}
....@@ -173,6 +180,28 @@
173180 		return true; /* coalesced */
174181 
175182 	return false;
183
+}
184
+
185
+static void ioapic_lazy_update_eoi(struct kvm_ioapic *ioapic, int irq)
186
+{
187
+	int i;
188
+	struct kvm_vcpu *vcpu;
189
+	union kvm_ioapic_redirect_entry *entry = &ioapic->redirtbl[irq];
190
+
191
+	kvm_for_each_vcpu(i, vcpu, ioapic->kvm) {
192
+		if (!kvm_apic_match_dest(vcpu, NULL, APIC_DEST_NOSHORT,
193
+					 entry->fields.dest_id,
194
+					 entry->fields.dest_mode) ||
195
+		    kvm_apic_pending_eoi(vcpu, entry->fields.vector))
196
+			continue;
197
+
198
+		/*
199
+		 * If no longer has pending EOI in LAPICs, update
200
+		 * EOI for this vector.
201
+		 */
202
+		rtc_irq_eoi(ioapic, vcpu, entry->fields.vector);
203
+		break;
204
+	}
176205 }
177206 
178207 static int ioapic_set_irq(struct kvm_ioapic *ioapic, unsigned int irq,
....@@ -193,9 +222,18 @@
193222 	}
194223 
195224 	/*
225
+	 * AMD SVM AVIC accelerate EOI write iff the interrupt is edge
226
+	 * triggered, in which case the in-kernel IOAPIC will not be able
227
+	 * to receive the EOI.  In this case, we do a lazy update of the
228
+	 * pending EOI when trying to set IOAPIC irq.
229
+	 */
230
+	if (edge && kvm_apicv_activated(ioapic->kvm))
231
+		ioapic_lazy_update_eoi(ioapic, irq);
232
+
233
+	/*
196234 	 * Return 0 for coalesced interrupts; for edge-triggered interrupts,
197235 	 * this only happens if a previous edge has not been delivered due
198
-	 * do masking.  For level interrupts, the remote_irr field tells
236
+	 * to masking.  For level interrupts, the remote_irr field tells
199237 	 * us if the interrupt is waiting for an EOI.
200238 	 *
201239 	 * RTC is special: it is edge-triggered, but userspace likes to know
....@@ -257,8 +295,10 @@
257295 		if (e->fields.trig_mode == IOAPIC_LEVEL_TRIG ||
258296 		    kvm_irq_has_notifier(ioapic->kvm, KVM_IRQCHIP_IOAPIC, index) ||
259297 		    index == RTC_GSI) {
260
-			if (kvm_apic_match_dest(vcpu, NULL, 0,
261
-			             e->fields.dest_id, e->fields.dest_mode) ||
298
+			u16 dm = kvm_lapic_irq_dest_mode(!!e->fields.dest_mode);
299
+
300
+			if (kvm_apic_match_dest(vcpu, NULL, APIC_DEST_NOSHORT,
301
+						e->fields.dest_id, dm) ||
262302 			    kvm_apic_pending_eoi(vcpu, e->fields.vector))
263303 				__set_bit(e->fields.vector,
264304 					  ioapic_handled_vectors);
....@@ -278,8 +318,9 @@
278318 {
279319 	unsigned index;
280320 	bool mask_before, mask_after;
281
-	int old_remote_irr, old_delivery_status;
282321 	union kvm_ioapic_redirect_entry *e;
322
+	int old_remote_irr, old_delivery_status, old_dest_id, old_dest_mode;
323
+	DECLARE_BITMAP(vcpu_bitmap, KVM_MAX_VCPUS);
283324 
284325 	switch (ioapic->ioregsel) {
285326 	case IOAPIC_REG_VERSION:
....@@ -296,7 +337,6 @@
296337 	default:
297338 		index = (ioapic->ioregsel - 0x10) >> 1;
298339 
299
-		ioapic_debug("change redir index %x val %x\n", index, val);
300340 		if (index >= IOAPIC_NUM_PINS)
301341 			return;
302342 		index = array_index_nospec(index, IOAPIC_NUM_PINS);
....@@ -305,6 +345,8 @@
305345 		/* Preserve read-only fields */
306346 		old_remote_irr = e->fields.remote_irr;
307347 		old_delivery_status = e->fields.delivery_status;
348
+		old_dest_id = e->fields.dest_id;
349
+		old_dest_mode = e->fields.dest_mode;
308350 		if (ioapic->ioregsel & 1) {
309351 			e->bits &= 0xffffffff;
310352 			e->bits |= (u64) val << 32;
....@@ -330,7 +372,40 @@
330372 		if (e->fields.trig_mode == IOAPIC_LEVEL_TRIG
331373 		    && ioapic->irr & (1 << index))
332374 			ioapic_service(ioapic, index, false);
333
-		kvm_make_scan_ioapic_request(ioapic->kvm);
375
+		if (e->fields.delivery_mode == APIC_DM_FIXED) {
376
+			struct kvm_lapic_irq irq;
377
+
378
+			irq.vector = e->fields.vector;
379
+			irq.delivery_mode = e->fields.delivery_mode << 8;
380
+			irq.dest_mode =
381
+			    kvm_lapic_irq_dest_mode(!!e->fields.dest_mode);
382
+			irq.level = false;
383
+			irq.trig_mode = e->fields.trig_mode;
384
+			irq.shorthand = APIC_DEST_NOSHORT;
385
+			irq.dest_id = e->fields.dest_id;
386
+			irq.msi_redir_hint = false;
387
+			bitmap_zero(vcpu_bitmap, KVM_MAX_VCPUS);
388
+			kvm_bitmap_or_dest_vcpus(ioapic->kvm, &irq,
389
+						 vcpu_bitmap);
390
+			if (old_dest_mode != e->fields.dest_mode ||
391
+			    old_dest_id != e->fields.dest_id) {
392
+				/*
393
+				 * Update vcpu_bitmap with vcpus specified in
394
+				 * the previous request as well. This is done to
395
+				 * keep ioapic_handled_vectors synchronized.
396
+				 */
397
+				irq.dest_id = old_dest_id;
398
+				irq.dest_mode =
399
+				    kvm_lapic_irq_dest_mode(
400
+					!!e->fields.dest_mode);
401
+				kvm_bitmap_or_dest_vcpus(ioapic->kvm, &irq,
402
+							 vcpu_bitmap);
403
+			}
404
+			kvm_make_scan_ioapic_request_mask(ioapic->kvm,
405
+							  vcpu_bitmap);
406
+		} else {
407
+			kvm_make_scan_ioapic_request(ioapic->kvm);
408
+		}
334409 		break;
335410 	}
336411 }
....@@ -346,19 +421,13 @@
346421 	    entry->fields.remote_irr))
347422 		return -1;
348423 
349
-	ioapic_debug("dest=%x dest_mode=%x delivery_mode=%x "
350
-		     "vector=%x trig_mode=%x\n",
351
-		     entry->fields.dest_id, entry->fields.dest_mode,
352
-		     entry->fields.delivery_mode, entry->fields.vector,
353
-		     entry->fields.trig_mode);
354
-
355424 	irqe.dest_id = entry->fields.dest_id;
356425 	irqe.vector = entry->fields.vector;
357
-	irqe.dest_mode = entry->fields.dest_mode;
426
+	irqe.dest_mode = kvm_lapic_irq_dest_mode(!!entry->fields.dest_mode);
358427 	irqe.trig_mode = entry->fields.trig_mode;
359428 	irqe.delivery_mode = entry->fields.delivery_mode << 8;
360429 	irqe.level = 1;
361
-	irqe.shorthand = 0;
430
+	irqe.shorthand = APIC_DEST_NOSHORT;
362431 	irqe.msi_redir_hint = false;
363432 
364433 	if (irqe.trig_mode == IOAPIC_EDGE_TRIG)
....@@ -430,72 +499,68 @@
430499 }
431500 
432501 #define IOAPIC_SUCCESSIVE_IRQ_MAX_COUNT 10000
433
-
434
-static void __kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu,
435
-			struct kvm_ioapic *ioapic, int vector, int trigger_mode)
502
+static void kvm_ioapic_update_eoi_one(struct kvm_vcpu *vcpu,
503
+				      struct kvm_ioapic *ioapic,
504
+				      int trigger_mode,
505
+				      int pin)
436506 {
437
-	struct dest_map *dest_map = &ioapic->rtc_status.dest_map;
438507 	struct kvm_lapic *apic = vcpu->arch.apic;
439
-	int i;
508
+	union kvm_ioapic_redirect_entry *ent = &ioapic->redirtbl[pin];
440509 
441
-	/* RTC special handling */
442
-	if (test_bit(vcpu->vcpu_id, dest_map->map) &&
443
-	    vector == dest_map->vectors[vcpu->vcpu_id])
444
-		rtc_irq_eoi(ioapic, vcpu);
510
+	/*
511
+	 * We are dropping lock while calling ack notifiers because ack
512
+	 * notifier callbacks for assigned devices call into IOAPIC
513
+	 * recursively. Since remote_irr is cleared only after call
514
+	 * to notifiers if the same vector will be delivered while lock
515
+	 * is dropped it will be put into irr and will be delivered
516
+	 * after ack notifier returns.
517
+	 */
518
+	spin_unlock(&ioapic->lock);
519
+	kvm_notify_acked_irq(ioapic->kvm, KVM_IRQCHIP_IOAPIC, pin);
520
+	spin_lock(&ioapic->lock);
445521 
446
-	for (i = 0; i < IOAPIC_NUM_PINS; i++) {
447
-		union kvm_ioapic_redirect_entry *ent = &ioapic->redirtbl[i];
522
+	if (trigger_mode != IOAPIC_LEVEL_TRIG ||
523
+	    kvm_lapic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI)
524
+		return;
448525 
449
-		if (ent->fields.vector != vector)
450
-			continue;
451
-
452
-		/*
453
-		 * We are dropping lock while calling ack notifiers because ack
454
-		 * notifier callbacks for assigned devices call into IOAPIC
455
-		 * recursively. Since remote_irr is cleared only after call
456
-		 * to notifiers if the same vector will be delivered while lock
457
-		 * is dropped it will be put into irr and will be delivered
458
-		 * after ack notifier returns.
459
-		 */
460
-		spin_unlock(&ioapic->lock);
461
-		kvm_notify_acked_irq(ioapic->kvm, KVM_IRQCHIP_IOAPIC, i);
462
-		spin_lock(&ioapic->lock);
463
-
464
-		if (trigger_mode != IOAPIC_LEVEL_TRIG ||
465
-		    kvm_lapic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI)
466
-			continue;
467
-
468
-		ASSERT(ent->fields.trig_mode == IOAPIC_LEVEL_TRIG);
469
-		ent->fields.remote_irr = 0;
470
-		if (!ent->fields.mask && (ioapic->irr & (1 << i))) {
471
-			++ioapic->irq_eoi[i];
472
-			if (ioapic->irq_eoi[i] == IOAPIC_SUCCESSIVE_IRQ_MAX_COUNT) {
473
-				/*
474
-				 * Real hardware does not deliver the interrupt
475
-				 * immediately during eoi broadcast, and this
476
-				 * lets a buggy guest make slow progress
477
-				 * even if it does not correctly handle a
478
-				 * level-triggered interrupt.  Emulate this
479
-				 * behavior if we detect an interrupt storm.
480
-				 */
481
-				schedule_delayed_work(&ioapic->eoi_inject, HZ / 100);
482
-				ioapic->irq_eoi[i] = 0;
483
-				trace_kvm_ioapic_delayed_eoi_inj(ent->bits);
484
-			} else {
485
-				ioapic_service(ioapic, i, false);
486
-			}
526
+	ASSERT(ent->fields.trig_mode == IOAPIC_LEVEL_TRIG);
527
+	ent->fields.remote_irr = 0;
528
+	if (!ent->fields.mask && (ioapic->irr & (1 << pin))) {
529
+		++ioapic->irq_eoi[pin];
530
+		if (ioapic->irq_eoi[pin] == IOAPIC_SUCCESSIVE_IRQ_MAX_COUNT) {
531
+			/*
532
+			 * Real hardware does not deliver the interrupt
533
+			 * immediately during eoi broadcast, and this
534
+			 * lets a buggy guest make slow progress
535
+			 * even if it does not correctly handle a
536
+			 * level-triggered interrupt.  Emulate this
537
+			 * behavior if we detect an interrupt storm.
538
+			 */
539
+			schedule_delayed_work(&ioapic->eoi_inject, HZ / 100);
540
+			ioapic->irq_eoi[pin] = 0;
541
+			trace_kvm_ioapic_delayed_eoi_inj(ent->bits);
487542 		} else {
488
-			ioapic->irq_eoi[i] = 0;
543
+			ioapic_service(ioapic, pin, false);
489544 		}
545
+	} else {
546
+		ioapic->irq_eoi[pin] = 0;
490547 	}
491548 }
492549 
493550 void kvm_ioapic_update_eoi(struct kvm_vcpu *vcpu, int vector, int trigger_mode)
494551 {
552
+	int i;
495553 	struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
496554 
497555 	spin_lock(&ioapic->lock);
498
-	__kvm_ioapic_update_eoi(vcpu, ioapic, vector, trigger_mode);
556
+	rtc_irq_eoi(ioapic, vcpu, vector);
557
+	for (i = 0; i < IOAPIC_NUM_PINS; i++) {
558
+		union kvm_ioapic_redirect_entry *ent = &ioapic->redirtbl[i];
559
+
560
+		if (ent->fields.vector != vector)
561
+			continue;
562
+		kvm_ioapic_update_eoi_one(vcpu, ioapic, trigger_mode, i);
563
+	}
499564 	spin_unlock(&ioapic->lock);
500565 }
501566 
....@@ -518,7 +583,6 @@
518583 	if (!ioapic_in_range(ioapic, addr))
519584 		return -EOPNOTSUPP;
520585 
521
-	ioapic_debug("addr %lx\n", (unsigned long)addr);
522586 	ASSERT(!(addr & 0xf));	/* check alignment */
523587 
524588 	addr &= 0xff;
....@@ -561,8 +625,6 @@
561625 	if (!ioapic_in_range(ioapic, addr))
562626 		return -EOPNOTSUPP;
563627 
564
-	ioapic_debug("ioapic_mmio_write addr=%p len=%d val=%p\n",
565
-		     (void*)addr, len, val);
566628 	ASSERT(!(addr & 0xf));	/* check alignment */
567629 
568630 	switch (len) {
....@@ -625,7 +687,7 @@
625687 	struct kvm_ioapic *ioapic;
626688 	int ret;
627689 
628
-	ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL);
690
+	ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL_ACCOUNT);
629691 	if (!ioapic)
630692 		return -ENOMEM;
631693 	spin_lock_init(&ioapic->lock);