forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/include/linux/intel-iommu.h
....@@ -1,22 +1,10 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 /*
23  * Copyright © 2006-2015, Intel Corporation.
34  *
45  * Authors: Ashok Raj <ashok.raj@intel.com>
56  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
67  *          David Woodhouse <David.Woodhouse@intel.com>
7
- *
8
- * This program is free software; you can redistribute it and/or modify it
9
- * under the terms and conditions of the GNU General Public License,
10
- * version 2, as published by the Free Software Foundation.
11
- *
12
- * This program is distributed in the hope it will be useful, but WITHOUT
13
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15
- * more details.
16
- *
17
- * You should have received a copy of the GNU General Public License along with
18
- * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
19
- * Place - Suite 330, Boston, MA 02111-1307 USA.
208  */
219 
2210 #ifndef _INTEL_IOMMU_H_
....@@ -26,20 +14,49 @@
2614 #include <linux/iova.h>
2715 #include <linux/io.h>
2816 #include <linux/idr.h>
29
-#include <linux/dma_remapping.h>
3017 #include <linux/mmu_notifier.h>
3118 #include <linux/list.h>
3219 #include <linux/iommu.h>
3320 #include <linux/io-64-nonatomic-lo-hi.h>
3421 #include <linux/dmar.h>
22
+#include <linux/ioasid.h>
3523 
3624 #include <asm/cacheflush.h>
3725 #include <asm/iommu.h>
3826 
3927 /*
28
+ * VT-d hardware uses 4KiB page size regardless of host page size.
29
+ */
30
+#define VTD_PAGE_SHIFT		(12)
31
+#define VTD_PAGE_SIZE		(1UL << VTD_PAGE_SHIFT)
32
+#define VTD_PAGE_MASK		(((u64)-1) << VTD_PAGE_SHIFT)
33
+#define VTD_PAGE_ALIGN(addr)	(((addr) + VTD_PAGE_SIZE - 1) & VTD_PAGE_MASK)
34
+
35
+#define VTD_STRIDE_SHIFT        (9)
36
+#define VTD_STRIDE_MASK         (((u64)-1) << VTD_STRIDE_SHIFT)
37
+
38
+#define DMA_PTE_READ		BIT_ULL(0)
39
+#define DMA_PTE_WRITE		BIT_ULL(1)
40
+#define DMA_PTE_LARGE_PAGE	BIT_ULL(7)
41
+#define DMA_PTE_SNP		BIT_ULL(11)
42
+
43
+#define DMA_FL_PTE_PRESENT	BIT_ULL(0)
44
+#define DMA_FL_PTE_US		BIT_ULL(2)
45
+#define DMA_FL_PTE_ACCESS	BIT_ULL(5)
46
+#define DMA_FL_PTE_DIRTY	BIT_ULL(6)
47
+#define DMA_FL_PTE_XD		BIT_ULL(63)
48
+
49
+#define ADDR_WIDTH_5LEVEL	(57)
50
+#define ADDR_WIDTH_4LEVEL	(48)
51
+
52
+#define CONTEXT_TT_MULTI_LEVEL	0
53
+#define CONTEXT_TT_DEV_IOTLB	1
54
+#define CONTEXT_TT_PASS_THROUGH 2
55
+#define CONTEXT_PASIDE		BIT_ULL(3)
56
+
57
+/*
4058  * Intel IOMMU register specification per version 1.0 public spec.
4159  */
42
-
4360 #define	DMAR_VER_REG	0x0	/* Arch version supported by this IOMMU */
4461 #define	DMAR_CAP_REG	0x8	/* Hardware supported capabilities */
4562 #define	DMAR_ECAP_REG	0x10	/* Extended capabilities supported */
....@@ -72,11 +89,49 @@
7289 #define	DMAR_PEDATA_REG	0xe4	/* Page request event interrupt data register */
7390 #define	DMAR_PEADDR_REG	0xe8	/* Page request event interrupt addr register */
7491 #define	DMAR_PEUADDR_REG 0xec	/* Page request event Upper address register */
92
+#define DMAR_MTRRCAP_REG 0x100	/* MTRR capability register */
93
+#define DMAR_MTRRDEF_REG 0x108	/* MTRR default type register */
94
+#define DMAR_MTRR_FIX64K_00000_REG 0x120 /* MTRR Fixed range registers */
95
+#define DMAR_MTRR_FIX16K_80000_REG 0x128
96
+#define DMAR_MTRR_FIX16K_A0000_REG 0x130
97
+#define DMAR_MTRR_FIX4K_C0000_REG 0x138
98
+#define DMAR_MTRR_FIX4K_C8000_REG 0x140
99
+#define DMAR_MTRR_FIX4K_D0000_REG 0x148
100
+#define DMAR_MTRR_FIX4K_D8000_REG 0x150
101
+#define DMAR_MTRR_FIX4K_E0000_REG 0x158
102
+#define DMAR_MTRR_FIX4K_E8000_REG 0x160
103
+#define DMAR_MTRR_FIX4K_F0000_REG 0x168
104
+#define DMAR_MTRR_FIX4K_F8000_REG 0x170
105
+#define DMAR_MTRR_PHYSBASE0_REG 0x180 /* MTRR Variable range registers */
106
+#define DMAR_MTRR_PHYSMASK0_REG 0x188
107
+#define DMAR_MTRR_PHYSBASE1_REG 0x190
108
+#define DMAR_MTRR_PHYSMASK1_REG 0x198
109
+#define DMAR_MTRR_PHYSBASE2_REG 0x1a0
110
+#define DMAR_MTRR_PHYSMASK2_REG 0x1a8
111
+#define DMAR_MTRR_PHYSBASE3_REG 0x1b0
112
+#define DMAR_MTRR_PHYSMASK3_REG 0x1b8
113
+#define DMAR_MTRR_PHYSBASE4_REG 0x1c0
114
+#define DMAR_MTRR_PHYSMASK4_REG 0x1c8
115
+#define DMAR_MTRR_PHYSBASE5_REG 0x1d0
116
+#define DMAR_MTRR_PHYSMASK5_REG 0x1d8
117
+#define DMAR_MTRR_PHYSBASE6_REG 0x1e0
118
+#define DMAR_MTRR_PHYSMASK6_REG 0x1e8
119
+#define DMAR_MTRR_PHYSBASE7_REG 0x1f0
120
+#define DMAR_MTRR_PHYSMASK7_REG 0x1f8
121
+#define DMAR_MTRR_PHYSBASE8_REG 0x200
122
+#define DMAR_MTRR_PHYSMASK8_REG 0x208
123
+#define DMAR_MTRR_PHYSBASE9_REG 0x210
124
+#define DMAR_MTRR_PHYSMASK9_REG 0x218
125
+#define DMAR_VCCAP_REG		0xe30 /* Virtual command capability register */
126
+#define DMAR_VCMD_REG		0xe00 /* Virtual command register */
127
+#define DMAR_VCRSP_REG		0xe10 /* Virtual command response register */
75128 
76129 #define OFFSET_STRIDE		(9)
77130 
78131 #define dmar_readq(a) readq(a)
79132 #define dmar_writeq(a,v) writeq(v,a)
133
+#define dmar_readl(a) readl(a)
134
+#define dmar_writel(a, v) writel(v, a)
80135 
81136 #define DMAR_VER_MAJOR(v)		(((v) & 0xf0) >> 4)
82137 #define DMAR_VER_MINOR(v)		((v) & 0x0f)
....@@ -115,6 +170,11 @@
115170  * Extended Capability Register
116171  */
117172 
173
+#define ecap_smpwc(e)		(((e) >> 48) & 0x1)
174
+#define ecap_flts(e)		(((e) >> 47) & 0x1)
175
+#define ecap_slts(e)		(((e) >> 46) & 0x1)
176
+#define ecap_vcs(e)		(((e) >> 44) & 0x1)
177
+#define ecap_smts(e)		(((e) >> 43) & 0x1)
118178 #define ecap_dit(e)		((e >> 41) & 0x1)
119179 #define ecap_pasid(e)		((e >> 40) & 0x1)
120180 #define ecap_pss(e)		((e >> 35) & 0x1f)
....@@ -138,6 +198,9 @@
138198 #define ecap_dev_iotlb_support(e)	(((e) >> 2) & 0x1)
139199 #define ecap_max_handle_mask(e) ((e >> 20) & 0xf)
140200 #define ecap_sc_support(e)	((e >> 7) & 0x1) /* Snooping Control */
201
+
202
+/* Virtual command interface capability */
203
+#define vccap_pasid(v)		(((v) & DMA_VCS_PAS)) /* PASID allocation */
141204 
142205 /* IOTLB_REG */
143206 #define DMA_TLB_FLUSH_GRANU_OFFSET  60
....@@ -193,6 +256,7 @@
193256 
194257 /* DMA_RTADDR_REG */
195258 #define DMA_RTADDR_RTT (((u64)1) << 11)
259
+#define DMA_RTADDR_SMT (((u64)1) << 10)
196260 
197261 /* CCMD_REG */
198262 #define DMA_CCMD_ICC (((u64)1) << 63)
....@@ -224,11 +288,16 @@
224288 #define dma_frcd_type(d) ((d >> 30) & 1)
225289 #define dma_frcd_fault_reason(c) (c & 0xff)
226290 #define dma_frcd_source_id(c) (c & 0xffff)
291
+#define dma_frcd_pasid_value(c) (((c) >> 8) & 0xfffff)
292
+#define dma_frcd_pasid_present(c) (((c) >> 31) & 1)
227293 /* low 64 bit */
228294 #define dma_frcd_page_addr(d) (d & (((u64)-1) << PAGE_SHIFT))
229295 
230296 /* PRS_REG */
231297 #define DMA_PRS_PPR	((u32)1)
298
+#define DMA_PRS_PRO	((u32)2)
299
+
300
+#define DMA_VCS_PAS	((u64)1)
232301 
233302 #define IOMMU_WAIT_OP(iommu, offset, op, cond, sts)			\
234303 do {									\
....@@ -269,6 +338,8 @@
269338 
270339 #define QI_IWD_STATUS_DATA(d)	(((u64)d) << 32)
271340 #define QI_IWD_STATUS_WRITE	(((u64)1) << 5)
341
+#define QI_IWD_FENCE		(((u64)1) << 6)
342
+#define QI_IWD_PRQ_DRAIN	(((u64)1) << 7)
272343 
273344 #define QI_IOTLB_DID(did) 	(((u64)did) << 16)
274345 #define QI_IOTLB_DR(dr) 	(((u64)dr) << 7)
....@@ -276,7 +347,7 @@
276347 #define QI_IOTLB_GRAN(gran) 	(((u64)gran) >> (DMA_TLB_FLUSH_GRANU_OFFSET-4))
277348 #define QI_IOTLB_ADDR(addr)	(((u64)addr) & VTD_PAGE_MASK)
278349 #define QI_IOTLB_IH(ih)		(((u64)ih) << 6)
279
-#define QI_IOTLB_AM(am)		(((u8)am))
350
+#define QI_IOTLB_AM(am)		(((u8)am) & 0x3f)
280351 
281352 #define QI_CC_FM(fm)		(((u64)fm) << 48)
282353 #define QI_CC_SID(sid)		(((u64)sid) << 32)
....@@ -295,20 +366,24 @@
295366 #define QI_PC_DID(did)		(((u64)did) << 16)
296367 #define QI_PC_GRAN(gran)	(((u64)gran) << 4)
297368 
298
-#define QI_PC_ALL_PASIDS	(QI_PC_TYPE | QI_PC_GRAN(0))
299
-#define QI_PC_PASID_SEL		(QI_PC_TYPE | QI_PC_GRAN(1))
369
+/* PASID cache invalidation granu */
370
+#define QI_PC_ALL_PASIDS	0
371
+#define QI_PC_PASID_SEL		1
372
+#define QI_PC_GLOBAL		3
300373 
301374 #define QI_EIOTLB_ADDR(addr)	((u64)(addr) & VTD_PAGE_MASK)
302
-#define QI_EIOTLB_GL(gl)	(((u64)gl) << 7)
303375 #define QI_EIOTLB_IH(ih)	(((u64)ih) << 6)
304
-#define QI_EIOTLB_AM(am)	(((u64)am))
376
+#define QI_EIOTLB_AM(am)	(((u64)am) & 0x3f)
305377 #define QI_EIOTLB_PASID(pasid) 	(((u64)pasid) << 32)
306378 #define QI_EIOTLB_DID(did)	(((u64)did) << 16)
307379 #define QI_EIOTLB_GRAN(gran) 	(((u64)gran) << 4)
308380 
381
+/* QI Dev-IOTLB inv granu */
382
+#define QI_DEV_IOTLB_GRAN_ALL		1
383
+#define QI_DEV_IOTLB_GRAN_PASID_SEL	0
384
+
309385 #define QI_DEV_EIOTLB_ADDR(a)	((u64)(a) & VTD_PAGE_MASK)
310386 #define QI_DEV_EIOTLB_SIZE	(((u64)1) << 11)
311
-#define QI_DEV_EIOTLB_GLOB(g)	((u64)(g) & 0x1)
312387 #define QI_DEV_EIOTLB_PASID(p)	((u64)((p) & 0xfffff) << 32)
313388 #define QI_DEV_EIOTLB_SID(sid)	((u64)((sid) & 0xffff) << 16)
314389 #define QI_DEV_EIOTLB_QDEP(qd)	((u64)((qd) & 0x1f) << 4)
....@@ -316,42 +391,44 @@
316391 				    ((u64)((pfsid >> 4) & 0xfff) << 52))
317392 #define QI_DEV_EIOTLB_MAX_INVS	32
318393 
319
-#define QI_PGRP_IDX(idx)	(((u64)(idx)) << 55)
320
-#define QI_PGRP_PRIV(priv)	(((u64)(priv)) << 32)
321
-#define QI_PGRP_RESP_CODE(res)	((u64)(res))
322
-#define QI_PGRP_PASID(pasid)	(((u64)(pasid)) << 32)
323
-#define QI_PGRP_DID(did)	(((u64)(did)) << 16)
394
+/* Page group response descriptor QW0 */
324395 #define QI_PGRP_PASID_P(p)	(((u64)(p)) << 4)
396
+#define QI_PGRP_PDP(p)		(((u64)(p)) << 5)
397
+#define QI_PGRP_RESP_CODE(res)	(((u64)(res)) << 12)
398
+#define QI_PGRP_DID(rid)	(((u64)(rid)) << 16)
399
+#define QI_PGRP_PASID(pasid)	(((u64)(pasid)) << 32)
325400 
326
-#define QI_PSTRM_ADDR(addr)	(((u64)(addr)) & VTD_PAGE_MASK)
327
-#define QI_PSTRM_DEVFN(devfn)	(((u64)(devfn)) << 4)
328
-#define QI_PSTRM_RESP_CODE(res)	((u64)(res))
329
-#define QI_PSTRM_IDX(idx)	(((u64)(idx)) << 55)
330
-#define QI_PSTRM_PRIV(priv)	(((u64)(priv)) << 32)
331
-#define QI_PSTRM_BUS(bus)	(((u64)(bus)) << 24)
332
-#define QI_PSTRM_PASID(pasid)	(((u64)(pasid)) << 4)
401
+/* Page group response descriptor QW1 */
402
+#define QI_PGRP_LPIG(x)		(((u64)(x)) << 2)
403
+#define QI_PGRP_IDX(idx)	(((u64)(idx)) << 3)
404
+
333405 
334406 #define QI_RESP_SUCCESS		0x0
335407 #define QI_RESP_INVALID		0x1
336408 #define QI_RESP_FAILURE		0xf
337409 
338
-#define QI_GRAN_ALL_ALL			0
339
-#define QI_GRAN_NONG_ALL		1
340410 #define QI_GRAN_NONG_PASID		2
341411 #define QI_GRAN_PSI_PASID		3
342412 
413
+#define qi_shift(iommu)		(DMAR_IQ_SHIFT + !!ecap_smts((iommu)->ecap))
414
+
343415 struct qi_desc {
344
-	u64 low, high;
416
+	u64 qw0;
417
+	u64 qw1;
418
+	u64 qw2;
419
+	u64 qw3;
345420 };
346421 
347422 struct q_inval {
348423 	raw_spinlock_t  q_lock;
349
-	struct qi_desc  *desc;          /* invalidation queue */
424
+	void		*desc;          /* invalidation queue */
350425 	int             *desc_status;   /* desc status */
351426 	int             free_head;      /* first free entry */
352427 	int             free_tail;      /* last free entry */
353428 	int             free_cnt;
354429 };
430
+
431
+struct dmar_pci_notify_info;
355432 
356433 #ifdef CONFIG_IRQ_REMAP
357434 /* 1MB - maximum possible interrupt remapping table size */
....@@ -367,6 +444,11 @@
367444 	struct irte *base;
368445 	unsigned long *bitmap;
369446 };
447
+
448
+void intel_irq_remap_add_device(struct dmar_pci_notify_info *info);
449
+#else
450
+static inline void
451
+intel_irq_remap_add_device(struct dmar_pci_notify_info *info) { }
370452 #endif
371453 
372454 struct iommu_flush {
....@@ -386,10 +468,62 @@
386468 
387469 #define VTD_FLAG_TRANS_PRE_ENABLED	(1 << 0)
388470 #define VTD_FLAG_IRQ_REMAP_PRE_ENABLED	(1 << 1)
471
+#define VTD_FLAG_SVM_CAPABLE		(1 << 2)
472
+
473
+extern int intel_iommu_sm;
474
+extern spinlock_t device_domain_lock;
475
+
476
+#define sm_supported(iommu)	(intel_iommu_sm && ecap_smts((iommu)->ecap))
477
+#define pasid_supported(iommu)	(sm_supported(iommu) &&			\
478
+				 ecap_pasid((iommu)->ecap))
389479 
390480 struct pasid_entry;
391481 struct pasid_state_entry;
392482 struct page_req_dsc;
483
+
484
+/*
485
+ * 0: Present
486
+ * 1-11: Reserved
487
+ * 12-63: Context Ptr (12 - (haw-1))
488
+ * 64-127: Reserved
489
+ */
490
+struct root_entry {
491
+	u64     lo;
492
+	u64     hi;
493
+};
494
+
495
+/*
496
+ * low 64 bits:
497
+ * 0: present
498
+ * 1: fault processing disable
499
+ * 2-3: translation type
500
+ * 12-63: address space root
501
+ * high 64 bits:
502
+ * 0-2: address width
503
+ * 3-6: aval
504
+ * 8-23: domain id
505
+ */
506
+struct context_entry {
507
+	u64 lo;
508
+	u64 hi;
509
+};
510
+
511
+/* si_domain contains mulitple devices */
512
+#define DOMAIN_FLAG_STATIC_IDENTITY		BIT(0)
513
+
514
+/*
515
+ * When VT-d works in the scalable mode, it allows DMA translation to
516
+ * happen through either first level or second level page table. This
517
+ * bit marks that the DMA translation for the domain goes through the
518
+ * first level page table, otherwise, it goes through the second level.
519
+ */
520
+#define DOMAIN_FLAG_USE_FIRST_LEVEL		BIT(1)
521
+
522
+/*
523
+ * Domain represents a virtual machine which demands iommu nested
524
+ * translation mode support.
525
+ */
526
+#define DOMAIN_FLAG_NESTING_MODE		BIT(2)
393527 
394528 struct dmar_domain {
395529 	int	nid;			/* node id */
....@@ -402,9 +536,11 @@
402536 					/* Domain ids per IOMMU. Use u16 since
403537 					 * domain ids are 16 bit wide according
404538 					 * to VT-d spec, section 9.3 */
539
+	unsigned int	auxd_refcnt;	/* Refcount of auxiliary attaching */
405540 
406541 	bool has_iotlb_device;
407542 	struct list_head devices;	/* all devices' list */
543
+	struct list_head auxd;		/* link to device's auxiliary list */
408544 	struct iova_domain iovad;	/* iova's that belong to this domain */
409545 
410546 	struct dma_pte	*pgd;		/* virtual address */
....@@ -423,6 +559,11 @@
423559 					   2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
424560 	u64		max_addr;	/* maximum mapped address */
425561 
562
+	u32		default_pasid;	/*
563
+					 * The default pasid used for non-SVM
564
+					 * traffic on mediated devices.
565
+					 */
566
+
426567 	struct iommu_domain domain;	/* generic domain data structure for
427568 					   iommu core */
428569 };
....@@ -433,6 +574,7 @@
433574 	u64		reg_size; /* size of hw register set */
434575 	u64		cap;
435576 	u64		ecap;
577
+	u64		vccap;
436578 	u32		gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */
437579 	raw_spinlock_t	register_lock; /* protect register handling */
438580 	int		seq_id;	/* sequence id of the iommu */
....@@ -451,15 +593,10 @@
451593 	struct iommu_flush flush;
452594 #endif
453595 #ifdef CONFIG_INTEL_IOMMU_SVM
454
-	/* These are large and need to be contiguous, so we allocate just
455
-	 * one for now. We'll maybe want to rethink that if we truly give
456
-	 * devices away to userspace processes (e.g. for DPDK) and don't
457
-	 * want to trust that userspace will use *only* the PASID it was
458
-	 * told to. But while it's all driver-arbitrated, we're fine. */
459
-	struct pasid_state_entry *pasid_state_table;
460596 	struct page_req_dsc *prq;
461597 	unsigned char prq_name[16];    /* Name for PRQ interrupt */
462
-	u32 pasid_max;
598
+	struct completion prq_complete;
599
+	struct ioasid_allocator_ops pasid_allocator; /* Custom allocator for PASIDs */
463600 #endif
464601 	struct q_inval  *qi;            /* Queued invalidation info */
465602 	u32 *iommu_state; /* Store iommu states between suspend and resume.*/
....@@ -481,6 +618,10 @@
481618 	struct list_head link;	/* link to domain siblings */
482619 	struct list_head global; /* link to global list */
483620 	struct list_head table;	/* link to pasid table */
621
+	struct list_head auxiliary_domains; /* auxiliary domains
622
+					     * attached to this device
623
+					     */
624
+	u32 segment;		/* PCI segment number */
484625 	u8 bus;			/* PCI bus number */
485626 	u8 devfn;		/* PCI devfn number */
486627 	u16 pfsid;		/* SRIOV physical function source ID */
....@@ -490,6 +631,7 @@
490631 	u8 pri_enabled:1;
491632 	u8 ats_supported:1;
492633 	u8 ats_enabled:1;
634
+	u8 auxd_enabled:1;	/* Multiple domains per device */
493635 	u8 ats_qdep;
494636 	struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
495637 	struct intel_iommu *iommu; /* IOMMU used by this device */
....@@ -502,6 +644,56 @@
502644 {
503645 	if (!ecap_coherent(iommu->ecap))
504646 		clflush_cache_range(addr, size);
647
+}
648
+
649
+/* Convert generic struct iommu_domain to private struct dmar_domain */
650
+static inline struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
651
+{
652
+	return container_of(dom, struct dmar_domain, domain);
653
+}
654
+
655
+/*
656
+ * 0: readable
657
+ * 1: writable
658
+ * 2-6: reserved
659
+ * 7: super page
660
+ * 8-10: available
661
+ * 11: snoop behavior
662
+ * 12-63: Host physcial address
663
+ */
664
+struct dma_pte {
665
+	u64 val;
666
+};
667
+
668
+static inline void dma_clear_pte(struct dma_pte *pte)
669
+{
670
+	pte->val = 0;
671
+}
672
+
673
+static inline u64 dma_pte_addr(struct dma_pte *pte)
674
+{
675
+#ifdef CONFIG_64BIT
676
+	return pte->val & VTD_PAGE_MASK & (~DMA_FL_PTE_XD);
677
+#else
678
+	/* Must have a full atomic 64-bit read */
679
+	return  __cmpxchg64(&pte->val, 0ULL, 0ULL) &
680
+			VTD_PAGE_MASK & (~DMA_FL_PTE_XD);
681
+#endif
682
+}
683
+
684
+static inline bool dma_pte_present(struct dma_pte *pte)
685
+{
686
+	return (pte->val & 3) != 0;
687
+}
688
+
689
+static inline bool dma_pte_superpage(struct dma_pte *pte)
690
+{
691
+	return (pte->val & DMA_PTE_LARGE_PAGE);
692
+}
693
+
694
+static inline int first_pte_in_page(struct dma_pte *pte)
695
+{
696
+	return !((unsigned long)pte & ~VTD_PAGE_MASK);
505697 }
506698 
507699 extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev);
....@@ -518,22 +710,50 @@
518710 			  unsigned int size_order, u64 type);
519711 extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
520712 			u16 qdep, u64 addr, unsigned mask);
521
-extern int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu);
713
+
714
+void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
715
+		     unsigned long npages, bool ih);
716
+
717
+void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
718
+			      u32 pasid, u16 qdep, u64 addr,
719
+			      unsigned int size_order);
720
+void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64 granu,
721
+			  u32 pasid);
722
+
723
+int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
724
+		   unsigned int count, unsigned long options);
725
+/*
726
+ * Options used in qi_submit_sync:
727
+ * QI_OPT_WAIT_DRAIN - Wait for PRQ drain completion, spec 6.5.2.8.
728
+ */
729
+#define QI_OPT_WAIT_DRAIN		BIT(0)
522730 
523731 extern int dmar_ir_support(void);
524732 
525
-struct dmar_domain *get_valid_domain_for_dev(struct device *dev);
526733 void *alloc_pgtable_page(int node);
527734 void free_pgtable_page(void *vaddr);
528735 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain);
529736 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
530737 				     void *data), void *data);
738
+void iommu_flush_write_buffer(struct intel_iommu *iommu);
739
+int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev);
740
+struct dmar_domain *find_domain(struct device *dev);
741
+struct device_domain_info *get_domain_info(struct device *dev);
742
+struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn);
531743 
532744 #ifdef CONFIG_INTEL_IOMMU_SVM
533
-int intel_svm_init(struct intel_iommu *iommu);
534
-int intel_svm_exit(struct intel_iommu *iommu);
745
+extern void intel_svm_check(struct intel_iommu *iommu);
535746 extern int intel_svm_enable_prq(struct intel_iommu *iommu);
536747 extern int intel_svm_finish_prq(struct intel_iommu *iommu);
748
+int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev,
749
+			  struct iommu_gpasid_bind_data *data);
750
+int intel_svm_unbind_gpasid(struct device *dev, u32 pasid);
751
+struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm,
752
+				 void *drvdata);
753
+void intel_svm_unbind(struct iommu_sva *handle);
754
+u32 intel_svm_get_pasid(struct iommu_sva *handle);
755
+int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt,
756
+			    struct iommu_page_response *msg);
537757 
538758 struct svm_dev_ops;
539759 
....@@ -541,7 +761,10 @@
541761 	struct list_head list;
542762 	struct rcu_head rcu;
543763 	struct device *dev;
764
+	struct intel_iommu *iommu;
544765 	struct svm_dev_ops *ops;
766
+	struct iommu_sva sva;
767
+	u32 pasid;
545768 	int users;
546769 	u16 did;
547770 	u16 dev_iotlb:1;
....@@ -551,17 +774,45 @@
551774 struct intel_svm {
552775 	struct mmu_notifier notifier;
553776 	struct mm_struct *mm;
554
-	struct intel_iommu *iommu;
555
-	int flags;
556
-	int pasid;
777
+
778
+	unsigned int flags;
779
+	u32 pasid;
780
+	int gpasid; /* In case that guest PASID is different from host PASID */
557781 	struct list_head devs;
558782 	struct list_head list;
559783 };
560
-
561
-extern int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev);
562
-extern struct intel_iommu *intel_svm_device_to_iommu(struct device *dev);
784
+#else
785
+static inline void intel_svm_check(struct intel_iommu *iommu) {}
563786 #endif
564787 
788
+#ifdef CONFIG_INTEL_IOMMU_DEBUGFS
789
+void intel_iommu_debugfs_init(void);
790
+#else
791
+static inline void intel_iommu_debugfs_init(void) {}
792
+#endif /* CONFIG_INTEL_IOMMU_DEBUGFS */
793
+
565794 extern const struct attribute_group *intel_iommu_groups[];
795
+bool context_present(struct context_entry *context);
796
+struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
797
+					 u8 devfn, int alloc);
798
+
799
+#ifdef CONFIG_INTEL_IOMMU
800
+extern int iommu_calculate_agaw(struct intel_iommu *iommu);
801
+extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu);
802
+extern int dmar_disabled;
803
+extern int intel_iommu_enabled;
804
+extern int intel_iommu_gfx_mapped;
805
+#else
806
+static inline int iommu_calculate_agaw(struct intel_iommu *iommu)
807
+{
808
+	return 0;
809
+}
810
+static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
811
+{
812
+	return 0;
813
+}
814
+#define dmar_disabled	(1)
815
+#define intel_iommu_enabled (0)
816
+#endif
566817 
567818 #endif