forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/include/linux/kvm_host.h
....@@ -1,10 +1,7 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 #ifndef __KVM_HOST_H
23 #define __KVM_HOST_H
34 
4
-/*
5
- * This work is licensed under the terms of the GNU GPL, version 2.  See
6
- * the COPYING file in the top-level directory.
7
- */
85 
96 #include <linux/types.h>
107 #include <linux/hardirq.h>
....@@ -26,7 +23,7 @@
2623 #include <linux/irqflags.h>
2724 #include <linux/context_tracking.h>
2825 #include <linux/irqbypass.h>
29
-#include <linux/swait.h>
26
+#include <linux/rcuwait.h>
3027 #include <linux/refcount.h>
3128 #include <linux/nospec.h>
3229 #include <asm/signal.h>
....@@ -48,6 +45,27 @@
4845  * include/linux/kvm_h.
4946  */
5047 #define KVM_MEMSLOT_INVALID	(1UL << 16)
48
+
49
+/*
50
+ * Bit 63 of the memslot generation number is an "update in-progress flag",
51
+ * e.g. is temporarily set for the duration of install_new_memslots().
52
+ * This flag effectively creates a unique generation number that is used to
53
+ * mark cached memslot data, e.g. MMIO accesses, as potentially being stale,
54
+ * i.e. may (or may not) have come from the previous memslots generation.
55
+ *
56
+ * This is necessary because the actual memslots update is not atomic with
57
+ * respect to the generation number update.  Updating the generation number
58
+ * first would allow a vCPU to cache a spte from the old memslots using the
59
+ * new generation number, and updating the generation number after switching
60
+ * to the new memslots would allow cache hits using the old generation number
61
+ * to reference the defunct memslots.
62
+ *
63
+ * This mechanism is used to prevent getting hits in KVM's caches while a
64
+ * memslot update is in-progress, and to prevent cache hits *after* updating
65
+ * the actual generation number against accesses that were inserted into the
66
+ * cache *before* the memslots were updated.
67
+ */
68
+#define KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS	BIT_ULL(63)
5169 
5270 /* Two fragments for cross MMIO pages. */
5371 #define KVM_MAX_MMIO_FRAGMENTS	2
....@@ -128,18 +146,17 @@
128146 #define KVM_REQ_MMU_RELOAD        (1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
129147 #define KVM_REQ_PENDING_TIMER     2
130148 #define KVM_REQ_UNHALT            3
149
+#define KVM_REQ_VM_BUGGED         (4 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
131150 #define KVM_REQUEST_ARCH_BASE     8
132151 
133152 #define KVM_ARCH_REQ_FLAGS(nr, flags) ({ \
134
-	BUILD_BUG_ON((unsigned)(nr) >= (FIELD_SIZEOF(struct kvm_vcpu, requests) * 8) - KVM_REQUEST_ARCH_BASE); \
153
+	BUILD_BUG_ON((unsigned)(nr) >= (sizeof_field(struct kvm_vcpu, requests) * 8) - KVM_REQUEST_ARCH_BASE); \
135154 	(unsigned)(((nr) + KVM_REQUEST_ARCH_BASE) | (flags)); \
136155 })
137156 #define KVM_ARCH_REQ(nr)           KVM_ARCH_REQ_FLAGS(nr, 0)
138157 
139158 #define KVM_USERSPACE_IRQ_SOURCE_ID		0
140159 #define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID	1
141
-
142
-extern struct kmem_cache *kvm_vcpu_cache;
143160 
144161 extern struct mutex kvm_lock;
145162 extern struct list_head vm_list;
....@@ -174,8 +191,8 @@
174191 		    int len, void *val);
175192 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
176193 			    int len, struct kvm_io_device *dev);
177
-void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
178
-			       struct kvm_io_device *dev);
194
+int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
195
+			      struct kvm_io_device *dev);
179196 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
180197 					 gpa_t addr);
181198 
....@@ -190,12 +207,13 @@
190207 	unsigned long addr;
191208 	struct kvm_arch_async_pf arch;
192209 	bool   wakeup_all;
210
+	bool notpresent_injected;
193211 };
194212 
195213 void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu);
196214 void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu);
197
-int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
198
-		       unsigned long hva, struct kvm_arch_async_pf *arch);
215
+bool kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
216
+			unsigned long hva, struct kvm_arch_async_pf *arch);
199217 int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu);
200218 #endif
201219 
....@@ -261,8 +279,7 @@
261279 	struct mutex mutex;
262280 	struct kvm_run *run;
263281 
264
-	int guest_xcr0_loaded;
265
-	struct swait_queue_head wq;
282
+	struct rcuwait wait;
266283 	struct pid __rcu *pid;
267284 	int sigset_active;
268285 	sigset_t sigset;
....@@ -301,8 +318,8 @@
301318 	} spin_loop;
302319 #endif
303320 	bool preempted;
321
+	bool ready;
304322 	struct kvm_vcpu_arch arch;
305
-	struct dentry *debugfs_dentry;
306323 };
307324 
308325 static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu)
....@@ -330,6 +347,7 @@
330347 	unsigned long userspace_addr;
331348 	u32 flags;
332349 	short id;
350
+	u16 as_id;
333351 };
334352 
335353 static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot)
....@@ -343,6 +361,10 @@
343361 
344362 	return memslot->dirty_bitmap + len / sizeof(*memslot->dirty_bitmap);
345363 }
364
+
365
+#ifndef KVM_DIRTY_LOG_MANUAL_CAPS
366
+#define KVM_DIRTY_LOG_MANUAL_CAPS KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE
367
+#endif
346368 
347369 struct kvm_s390_adapter_int {
348370 	u64 ind_addr;
....@@ -389,7 +411,7 @@
389411 	 * Array indexed by gsi. Each entry contains list of irq chips
390412 	 * the gsi is connected to.
391413 	 */
392
-	struct hlist_head map[0];
414
+	struct hlist_head map[];
393415 };
394416 #endif
395417 
....@@ -415,11 +437,11 @@
415437  */
416438 struct kvm_memslots {
417439 	u64 generation;
418
-	struct kvm_memory_slot memslots[KVM_MEM_SLOTS_NUM];
419440 	/* The mapping table from slot id to the index in memslots[]. */
420441 	short id_to_index[KVM_MEM_SLOTS_NUM];
421442 	atomic_t lru_slot;
422443 	int used_slots;
444
+	struct kvm_memory_slot memslots[];
423445 };
424446 
425447 struct kvm {
....@@ -477,11 +499,14 @@
477499 #endif
478500 	long tlbs_dirty;
479501 	struct list_head devices;
502
+	u64 manual_dirty_log_protect;
480503 	struct dentry *debugfs_dentry;
481504 	struct kvm_stat_data **debugfs_stat_data;
482505 	struct srcu_struct srcu;
483506 	struct srcu_struct irq_srcu;
484507 	pid_t userspace_pid;
508
+	unsigned int max_halt_poll_ns;
509
+	bool vm_bugged;
485510 };
486511 
487512 #define kvm_err(fmt, ...) \
....@@ -509,6 +534,36 @@
509534 			      ## __VA_ARGS__)
510535 #define vcpu_err(vcpu, fmt, ...)					\
511536 	kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
537
+
538
+bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req);
539
+static inline void kvm_vm_bugged(struct kvm *kvm)
540
+{
541
+	kvm->vm_bugged = true;
542
+	kvm_make_all_cpus_request(kvm, KVM_REQ_VM_BUGGED);
543
+}
544
+
545
+#define KVM_BUG(cond, kvm, fmt...)				\
546
+({								\
547
+	int __ret = (cond);					\
548
+								\
549
+	if (WARN_ONCE(__ret && !(kvm)->vm_bugged, fmt))		\
550
+		kvm_vm_bugged(kvm);				\
551
+	unlikely(__ret);					\
552
+})
553
+
554
+#define KVM_BUG_ON(cond, kvm)					\
555
+({								\
556
+	int __ret = (cond);					\
557
+								\
558
+	if (WARN_ON_ONCE(__ret && !(kvm)->vm_bugged))		\
559
+		kvm_vm_bugged(kvm);				\
560
+	unlikely(__ret);					\
561
+})
562
+
563
+static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm)
564
+{
565
+	return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET);
566
+}
512567 
513568 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)
514569 {
....@@ -555,13 +610,13 @@
555610 	return vcpu->vcpu_idx;
556611 }
557612 
558
-#define kvm_for_each_memslot(memslot, slots)	\
559
-	for (memslot = &slots->memslots[0];	\
560
-	      memslot < slots->memslots + KVM_MEM_SLOTS_NUM && memslot->npages;\
561
-		memslot++)
613
+#define kvm_for_each_memslot(memslot, slots)				\
614
+	for (memslot = &slots->memslots[0];				\
615
+	     memslot < slots->memslots + slots->used_slots; memslot++)	\
616
+		if (WARN_ON_ONCE(!memslot->npages)) {			\
617
+		} else
562618 
563
-int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id);
564
-void kvm_vcpu_uninit(struct kvm_vcpu *vcpu);
619
+void kvm_vcpu_destroy(struct kvm_vcpu *vcpu);
565620 
566621 void vcpu_load(struct kvm_vcpu *vcpu);
567622 void vcpu_put(struct kvm_vcpu *vcpu);
....@@ -597,6 +652,7 @@
597652 
598653 void kvm_get_kvm(struct kvm *kvm);
599654 void kvm_put_kvm(struct kvm *kvm);
655
+void kvm_put_kvm_no_destroy(struct kvm *kvm);
600656 
601657 static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id)
602658 {
....@@ -618,11 +674,14 @@
618674 	return __kvm_memslots(vcpu->kvm, as_id);
619675 }
620676 
621
-static inline struct kvm_memory_slot *
622
-id_to_memslot(struct kvm_memslots *slots, int id)
677
+static inline
678
+struct kvm_memory_slot *id_to_memslot(struct kvm_memslots *slots, int id)
623679 {
624680 	int index = slots->id_to_index[id];
625681 	struct kvm_memory_slot *slot;
682
+
683
+	if (index < 0)
684
+		return NULL;
626685 
627686 	slot = &slots->memslots[index];
628687 
....@@ -652,10 +711,7 @@
652711 			  const struct kvm_userspace_memory_region *mem);
653712 int __kvm_set_memory_region(struct kvm *kvm,
654713 			    const struct kvm_userspace_memory_region *mem);
655
-void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
656
-			   struct kvm_memory_slot *dont);
657
-int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
658
-			    unsigned long npages);
714
+void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot);
659715 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen);
660716 int kvm_arch_prepare_memory_region(struct kvm *kvm,
661717 				struct kvm_memory_slot *memslot,
....@@ -663,11 +719,9 @@
663719 				enum kvm_mr_change change);
664720 void kvm_arch_commit_memory_region(struct kvm *kvm,
665721 				const struct kvm_userspace_memory_region *mem,
666
-				const struct kvm_memory_slot *old,
722
+				struct kvm_memory_slot *old,
667723 				const struct kvm_memory_slot *new,
668724 				enum kvm_mr_change change);
669
-bool kvm_largepages_enabled(void);
670
-void kvm_disable_largepages(void);
671725 /* flush all memory translations */
672726 void kvm_arch_flush_shadow_all(struct kvm *kvm);
673727 /* flush memory translations pointing to 'slot' */
....@@ -687,7 +741,6 @@
687741 void kvm_release_page_dirty(struct page *page);
688742 void kvm_set_page_accessed(struct page *page);
689743 
690
-kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn);
691744 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
692745 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
693746 		      bool *writable);
....@@ -706,11 +759,12 @@
706759 void kvm_release_pfn(kvm_pfn_t pfn, bool dirty, struct gfn_to_pfn_cache *cache);
707760 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
708761 			int len);
709
-int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
710
-			  unsigned long len);
711762 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
712763 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
713764 			   void *data, unsigned long len);
765
+int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
766
+				 void *data, unsigned int offset,
767
+				 unsigned long len);
714768 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
715769 			 int offset, int len);
716770 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
....@@ -722,11 +776,56 @@
722776 				  unsigned long len);
723777 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
724778 			      gpa_t gpa, unsigned long len);
779
+
780
+#define __kvm_get_guest(kvm, gfn, offset, v)				\
781
+({									\
782
+	unsigned long __addr = gfn_to_hva(kvm, gfn);			\
783
+	typeof(v) __user *__uaddr = (typeof(__uaddr))(__addr + offset);	\
784
+	int __ret = -EFAULT;						\
785
+									\
786
+	if (!kvm_is_error_hva(__addr))					\
787
+		__ret = get_user(v, __uaddr);				\
788
+	__ret;								\
789
+})
790
+
791
+#define kvm_get_guest(kvm, gpa, v)					\
792
+({									\
793
+	gpa_t __gpa = gpa;						\
794
+	struct kvm *__kvm = kvm;					\
795
+									\
796
+	__kvm_get_guest(__kvm, __gpa >> PAGE_SHIFT,			\
797
+			offset_in_page(__gpa), v);			\
798
+})
799
+
800
+#define __kvm_put_guest(kvm, gfn, offset, v)				\
801
+({									\
802
+	unsigned long __addr = gfn_to_hva(kvm, gfn);			\
803
+	typeof(v) __user *__uaddr = (typeof(__uaddr))(__addr + offset);	\
804
+	int __ret = -EFAULT;						\
805
+									\
806
+	if (!kvm_is_error_hva(__addr))					\
807
+		__ret = put_user(v, __uaddr);				\
808
+	if (!__ret)							\
809
+		mark_page_dirty(kvm, gfn);				\
810
+	__ret;								\
811
+})
812
+
813
+#define kvm_put_guest(kvm, gpa, v)					\
814
+({									\
815
+	gpa_t __gpa = gpa;						\
816
+	struct kvm *__kvm = kvm;					\
817
+									\
818
+	__kvm_put_guest(__kvm, __gpa >> PAGE_SHIFT,			\
819
+			offset_in_page(__gpa), v);			\
820
+})
821
+
725822 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len);
726823 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
727824 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
728825 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
826
+bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
729827 unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn);
828
+void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
730829 void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
731830 
732831 struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu);
....@@ -768,9 +867,20 @@
768867 void kvm_flush_remote_tlbs(struct kvm *kvm);
769868 void kvm_reload_remote_mmus(struct kvm *kvm);
770869 
870
+#ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE
871
+int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min);
872
+int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc);
873
+void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc);
874
+void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc);
875
+#endif
876
+
771877 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
878
+				 struct kvm_vcpu *except,
772879 				 unsigned long *vcpu_bitmap, cpumask_var_t tmp);
773
-bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req);
880
+bool kvm_make_all_cpus_request_except(struct kvm *kvm, unsigned int req,
881
+				      struct kvm_vcpu *except);
882
+bool kvm_make_cpus_request_mask(struct kvm *kvm, unsigned int req,
883
+				unsigned long *vcpu_bitmap);
774884 
775885 long kvm_arch_dev_ioctl(struct file *filp,
776886 			unsigned int ioctl, unsigned long arg);
....@@ -780,24 +890,29 @@
780890 
781891 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext);
782892 
783
-int kvm_get_dirty_log(struct kvm *kvm,
784
-			struct kvm_dirty_log *log, int *is_dirty);
785
-
786
-int kvm_get_dirty_log_protect(struct kvm *kvm,
787
-			struct kvm_dirty_log *log, bool *is_dirty);
788
-
789893 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
790894 					struct kvm_memory_slot *slot,
791895 					gfn_t gfn_offset,
792896 					unsigned long mask);
897
+void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot);
793898 
794
-int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
795
-				struct kvm_dirty_log *log);
899
+#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
900
+void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
901
+					struct kvm_memory_slot *memslot);
902
+#else /* !CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
903
+int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log);
904
+int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,
905
+		      int *is_dirty, struct kvm_memory_slot **memslot);
906
+#endif
796907 
797908 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
798909 			bool line_status);
910
+int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
911
+			    struct kvm_enable_cap *cap);
799912 long kvm_arch_vm_ioctl(struct file *filp,
800913 		       unsigned int ioctl, unsigned long arg);
914
+long kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl,
915
+			      unsigned long arg);
801916 
802917 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
803918 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
....@@ -817,36 +932,35 @@
817932 				    struct kvm_mp_state *mp_state);
818933 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
819934 					struct kvm_guest_debug *dbg);
820
-int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
935
+int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu);
821936 
822937 int kvm_arch_init(void *opaque);
823938 void kvm_arch_exit(void);
824939 
825
-int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu);
826
-void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu);
827
-
828940 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu);
829941 
830
-void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu);
831942 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
832943 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu);
833
-struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id);
834
-int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu);
944
+int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id);
945
+int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu);
835946 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu);
836947 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu);
837948 
838
-bool kvm_arch_has_vcpu_debugfs(void);
839
-int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu);
949
+#ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS
950
+void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry);
951
+#endif
840952 
841953 int kvm_arch_hardware_enable(void);
842954 void kvm_arch_hardware_disable(void);
843
-int kvm_arch_hardware_setup(void);
955
+int kvm_arch_hardware_setup(void *opaque);
844956 void kvm_arch_hardware_unsetup(void);
845
-void kvm_arch_check_processor_compat(void *rtn);
957
+int kvm_arch_check_processor_compat(void *opaque);
846958 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
847959 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu);
848960 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
849961 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu);
962
+int kvm_arch_post_init_vm(struct kvm *kvm);
963
+void kvm_arch_pre_destroy_vm(struct kvm *kvm);
850964 
851965 #ifndef __KVM_HAVE_ARCH_VM_ALLOC
852966 /*
....@@ -902,18 +1016,18 @@
9021016 {
9031017 }
9041018 
905
-static inline bool kvm_arch_has_assigned_device(struct kvm *kvm)
1019
+static __always_inline bool kvm_arch_has_assigned_device(struct kvm *kvm)
9061020 {
9071021 	return false;
9081022 }
9091023 #endif
9101024 
911
-static inline struct swait_queue_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
1025
+static inline struct rcuwait *kvm_arch_vcpu_get_wait(struct kvm_vcpu *vcpu)
9121026 {
9131027 #ifdef __KVM_HAVE_ARCH_WQP
914
-	return vcpu->arch.wqp;
1028
+	return vcpu->arch.waitp;
9151029 #else
916
-	return &vcpu->wq;
1030
+	return &vcpu->wait;
9171031 #endif
9181032 }
9191033 
....@@ -936,10 +1050,10 @@
9361050 void kvm_arch_sync_events(struct kvm *kvm);
9371051 
9381052 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
939
-void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
9401053 
9411054 bool kvm_is_reserved_pfn(kvm_pfn_t pfn);
9421055 bool kvm_is_zone_device_pfn(kvm_pfn_t pfn);
1056
+bool kvm_is_transparent_hugepage(kvm_pfn_t pfn);
9431057 
9441058 struct kvm_irq_ack_notifier {
9451059 	struct hlist_node link;
....@@ -967,12 +1081,15 @@
9671081 				   struct kvm_irq_ack_notifier *kian);
9681082 int kvm_request_irq_source_id(struct kvm *kvm);
9691083 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
1084
+bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args);
9701085 
9711086 /*
9721087  * search_memslots() and __gfn_to_memslot() are here because they are
9731088  * used in non-modular code in arch/powerpc/kvm/book3s_hv_rm_mmu.c.
9741089  * gfn_to_memslot() itself isn't here as an inline because that would
9751090  * bloat other code too much.
1091
+ *
1092
+ * IMPORTANT: Slots are sorted from highest GFN to lowest GFN!
9761093  */
9771094 static inline struct kvm_memory_slot *
9781095 search_memslots(struct kvm_memslots *slots, gfn_t gfn)
....@@ -980,6 +1097,9 @@
9801097 	int start = 0, end = slots->used_slots;
9811098 	int slot = atomic_read(&slots->lru_slot);
9821099 	struct kvm_memory_slot *memslots = slots->memslots;
1100
+
1101
+	if (unlikely(!slots->used_slots))
1102
+		return NULL;
9831103 
9841104 	if (gfn >= memslots[slot].base_gfn &&
9851105 	    gfn < memslots[slot].base_gfn + memslots[slot].npages)
....@@ -1070,9 +1190,8 @@
10701190 };
10711191 
10721192 struct kvm_stat_data {
1073
-	int offset;
1074
-	int mode;
10751193 	struct kvm *kvm;
1194
+	struct kvm_stats_debugfs_item *dbgfs_item;
10761195 };
10771196 
10781197 struct kvm_stats_debugfs_item {
....@@ -1081,6 +1200,15 @@
10811200 	enum kvm_stat_kind kind;
10821201 	int mode;
10831202 };
1203
+
1204
+#define KVM_DBGFS_GET_MODE(dbgfs_item)                                         \
1205
+	((dbgfs_item)->mode ? (dbgfs_item)->mode : 0644)
1206
+
1207
+#define VM_STAT(n, x, ...) 							\
1208
+	{ n, offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ }
1209
+#define VCPU_STAT(n, x, ...)							\
1210
+	{ n, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ }
1211
+
10841212 extern struct kvm_stats_debugfs_item debugfs_entries[];
10851213 extern struct dentry *kvm_debugfs_dir;
10861214 
....@@ -1217,10 +1345,11 @@
12171345 
12181346 extern unsigned int halt_poll_ns;
12191347 extern unsigned int halt_poll_ns_grow;
1348
+extern unsigned int halt_poll_ns_grow_start;
12201349 extern unsigned int halt_poll_ns_shrink;
12211350 
12221351 struct kvm_device {
1223
-	struct kvm_device_ops *ops;
1352
+	const struct kvm_device_ops *ops;
12241353 	struct kvm *kvm;
12251354 	void *private;
12261355 	struct list_head vm_node;
....@@ -1253,17 +1382,27 @@
12531382 	 */
12541383 	void (*destroy)(struct kvm_device *dev);
12551384 
1385
+	/*
1386
+	 * Release is an alternative method to free the device. It is
1387
+	 * called when the device file descriptor is closed. Once
1388
+	 * release is called, the destroy method will not be called
1389
+	 * anymore as the device is removed from the device list of
1390
+	 * the VM. kvm->lock is held.
1391
+	 */
1392
+	void (*release)(struct kvm_device *dev);
1393
+
12561394 	int (*set_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
12571395 	int (*get_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
12581396 	int (*has_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
12591397 	long (*ioctl)(struct kvm_device *dev, unsigned int ioctl,
12601398 		      unsigned long arg);
1399
+	int (*mmap)(struct kvm_device *dev, struct vm_area_struct *vma);
12611400 };
12621401 
12631402 void kvm_device_get(struct kvm_device *dev);
12641403 void kvm_device_put(struct kvm_device *dev);
12651404 struct kvm_device *kvm_device_from_filp(struct file *filp);
1266
-int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type);
1405
+int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type);
12671406 void kvm_unregister_device_ops(u32 type);
12681407 
12691408 extern struct kvm_device_ops kvm_mpic_ops;
....@@ -1292,6 +1431,15 @@
12921431 }
12931432 #endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
12941433 
1434
+static inline bool kvm_is_visible_memslot(struct kvm_memory_slot *memslot)
1435
+{
1436
+	return (memslot && memslot->id < KVM_USER_MEM_SLOTS &&
1437
+		!(memslot->flags & KVM_MEMSLOT_INVALID));
1438
+}
1439
+
1440
+struct kvm_vcpu *kvm_get_running_vcpu(void);
1441
+struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
1442
+
12951443 #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS
12961444 bool kvm_arch_has_irq_bypass(void);
12971445 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *,
....@@ -1318,6 +1466,16 @@
13181466 }
13191467 #endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */
13201468 
1469
+#ifdef CONFIG_HAVE_KVM_NO_POLL
1470
+/* Callback that tells if we must not poll */
1471
+bool kvm_arch_no_poll(struct kvm_vcpu *vcpu);
1472
+#else
1473
+static inline bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
1474
+{
1475
+	return false;
1476
+}
1477
+#endif /* CONFIG_HAVE_KVM_NO_POLL */
1478
+
13211479 #ifdef CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL
13221480 long kvm_arch_vcpu_async_ioctl(struct file *filp,
13231481 			       unsigned int ioctl, unsigned long arg);
....@@ -1333,6 +1491,8 @@
13331491 void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
13341492 					    unsigned long start, unsigned long end);
13351493 
1494
+void kvm_arch_guest_memory_reclaimed(struct kvm *kvm);
1495
+
13361496 #ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE
13371497 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu);
13381498 #else
....@@ -1348,4 +1508,12 @@
13481508 				uintptr_t data, const char *name,
13491509 				struct task_struct **thread_ptr);
13501510 
1511
+#ifdef CONFIG_KVM_XFER_TO_GUEST_WORK
1512
+static inline void kvm_handle_signal_exit(struct kvm_vcpu *vcpu)
1513
+{
1514
+	vcpu->run->exit_reason = KVM_EXIT_INTR;
1515
+	vcpu->stat.signal_exits++;
1516
+}
1517
+#endif /* CONFIG_KVM_XFER_TO_GUEST_WORK */
1518
+
13511519 #endif