forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-13 9d77db3c730780c8ef5ccd4b66403ff5675cfe4e 
 kernel/drivers/gpu/drm/i915/i915_gem_gtt.h
....@@ -1,652 +1,21 @@
1
+/* SPDX-License-Identifier: MIT */
12 /*
2
- * Copyright © 2014 Intel Corporation
3
- *
4
- * Permission is hereby granted, free of charge, to any person obtaining a
5
- * copy of this software and associated documentation files (the "Software"),
6
- * to deal in the Software without restriction, including without limitation
7
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
- * and/or sell copies of the Software, and to permit persons to whom the
9
- * Software is furnished to do so, subject to the following conditions:
10
- *
11
- * The above copyright notice and this permission notice (including the next
12
- * paragraph) shall be included in all copies or substantial portions of the
13
- * Software.
14
- *
15
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21
- * IN THE SOFTWARE.
22
- *
23
- * Please try to maintain the following order within this file unless it makes
24
- * sense to do otherwise. From top to bottom:
25
- * 1. typedefs
26
- * 2. #defines, and macros
27
- * 3. structure definitions
28
- * 4. function prototypes
29
- *
30
- * Within each section, please try to order by generation in ascending order,
31
- * from top to bottom (ie. gen6 on the top, gen8 on the bottom).
3
+ * Copyright © 2020 Intel Corporation
324  */
335 
346 #ifndef __I915_GEM_GTT_H__
357 #define __I915_GEM_GTT_H__
368 
379 #include <linux/io-mapping.h>
38
-#include <linux/mm.h>
39
-#include <linux/pagevec.h>
10
+#include <linux/types.h>
4011 
41
-#include "i915_request.h"
42
-#include "i915_selftest.h"
43
-#include "i915_timeline.h"
12
+#include <drm/drm_mm.h>
4413 
45
-#define I915_GTT_PAGE_SIZE_4K	BIT_ULL(12)
46
-#define I915_GTT_PAGE_SIZE_64K	BIT_ULL(16)
47
-#define I915_GTT_PAGE_SIZE_2M	BIT_ULL(21)
14
+#include "gt/intel_gtt.h"
15
+#include "i915_scatterlist.h"
4816 
49
-#define I915_GTT_PAGE_SIZE I915_GTT_PAGE_SIZE_4K
50
-#define I915_GTT_MAX_PAGE_SIZE I915_GTT_PAGE_SIZE_2M
51
-
52
-#define I915_GTT_PAGE_MASK -I915_GTT_PAGE_SIZE
53
-
54
-#define I915_GTT_MIN_ALIGNMENT I915_GTT_PAGE_SIZE
55
-
56
-#define I915_FENCE_REG_NONE -1
57
-#define I915_MAX_NUM_FENCES 32
58
-/* 32 fences + sign bit for FENCE_REG_NONE */
59
-#define I915_MAX_NUM_FENCE_BITS 6
60
-
61
-struct drm_i915_file_private;
62
-struct drm_i915_fence_reg;
63
-struct i915_vma;
64
-
65
-typedef u32 gen6_pte_t;
66
-typedef u64 gen8_pte_t;
67
-typedef u64 gen8_pde_t;
68
-typedef u64 gen8_ppgtt_pdpe_t;
69
-typedef u64 gen8_ppgtt_pml4e_t;
70
-
71
-#define ggtt_total_entries(ggtt) ((ggtt)->vm.total >> PAGE_SHIFT)
72
-
73
-/* gen6-hsw has bit 11-4 for physical addr bit 39-32 */
74
-#define GEN6_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0xff0))
75
-#define GEN6_PTE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
76
-#define GEN6_PDE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
77
-#define GEN6_PTE_CACHE_LLC		(2 << 1)
78
-#define GEN6_PTE_UNCACHED		(1 << 1)
79
-#define GEN6_PTE_VALID			(1 << 0)
80
-
81
-#define I915_PTES(pte_len)		((unsigned int)(PAGE_SIZE / (pte_len)))
82
-#define I915_PTE_MASK(pte_len)		(I915_PTES(pte_len) - 1)
83
-#define I915_PDES			512
84
-#define I915_PDE_MASK			(I915_PDES - 1)
85
-#define NUM_PTE(pde_shift)     (1 << (pde_shift - PAGE_SHIFT))
86
-
87
-#define GEN6_PTES			I915_PTES(sizeof(gen6_pte_t))
88
-#define GEN6_PD_SIZE		        (I915_PDES * PAGE_SIZE)
89
-#define GEN6_PD_ALIGN			(PAGE_SIZE * 16)
90
-#define GEN6_PDE_SHIFT			22
91
-#define GEN6_PDE_VALID			(1 << 0)
92
-
93
-#define GEN7_PTE_CACHE_L3_LLC		(3 << 1)
94
-
95
-#define BYT_PTE_SNOOPED_BY_CPU_CACHES	(1 << 2)
96
-#define BYT_PTE_WRITEABLE		(1 << 1)
97
-
98
-/* Cacheability Control is a 4-bit value. The low three bits are stored in bits
99
- * 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
100
- */
101
-#define HSW_CACHEABILITY_CONTROL(bits)	((((bits) & 0x7) << 1) | \
102
-					 (((bits) & 0x8) << (11 - 3)))
103
-#define HSW_WB_LLC_AGE3			HSW_CACHEABILITY_CONTROL(0x2)
104
-#define HSW_WB_LLC_AGE0			HSW_CACHEABILITY_CONTROL(0x3)
105
-#define HSW_WB_ELLC_LLC_AGE3		HSW_CACHEABILITY_CONTROL(0x8)
106
-#define HSW_WB_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0xb)
107
-#define HSW_WT_ELLC_LLC_AGE3		HSW_CACHEABILITY_CONTROL(0x7)
108
-#define HSW_WT_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0x6)
109
-#define HSW_PTE_UNCACHED		(0)
110
-#define HSW_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0x7f0))
111
-#define HSW_PTE_ADDR_ENCODE(addr)	HSW_GTT_ADDR_ENCODE(addr)
112
-
113
-/* GEN8 32b style address is defined as a 3 level page table:
114
- * 31:30 | 29:21 | 20:12 |  11:0
115
- * PDPE  |  PDE  |  PTE  | offset
116
- * The difference as compared to normal x86 3 level page table is the PDPEs are
117
- * programmed via register.
118
- */
119
-#define GEN8_3LVL_PDPES			4
120
-#define GEN8_PDE_SHIFT			21
121
-#define GEN8_PDE_MASK			0x1ff
122
-#define GEN8_PTE_SHIFT			12
123
-#define GEN8_PTE_MASK			0x1ff
124
-#define GEN8_PTES			I915_PTES(sizeof(gen8_pte_t))
125
-
126
-/* GEN8 48b style address is defined as a 4 level page table:
127
- * 47:39 | 38:30 | 29:21 | 20:12 |  11:0
128
- * PML4E | PDPE  |  PDE  |  PTE  | offset
129
- */
130
-#define GEN8_PML4ES_PER_PML4		512
131
-#define GEN8_PML4E_SHIFT		39
132
-#define GEN8_PML4E_MASK			(GEN8_PML4ES_PER_PML4 - 1)
133
-#define GEN8_PDPE_SHIFT			30
134
-/* NB: GEN8_PDPE_MASK is untrue for 32b platforms, but it has no impact on 32b page
135
- * tables */
136
-#define GEN8_PDPE_MASK			0x1ff
137
-
138
-#define PPAT_UNCACHED			(_PAGE_PWT | _PAGE_PCD)
139
-#define PPAT_CACHED_PDE			0 /* WB LLC */
140
-#define PPAT_CACHED			_PAGE_PAT /* WB LLCeLLC */
141
-#define PPAT_DISPLAY_ELLC		_PAGE_PCD /* WT eLLC */
142
-
143
-#define CHV_PPAT_SNOOP			(1<<6)
144
-#define GEN8_PPAT_AGE(x)		((x)<<4)
145
-#define GEN8_PPAT_LLCeLLC		(3<<2)
146
-#define GEN8_PPAT_LLCELLC		(2<<2)
147
-#define GEN8_PPAT_LLC			(1<<2)
148
-#define GEN8_PPAT_WB			(3<<0)
149
-#define GEN8_PPAT_WT			(2<<0)
150
-#define GEN8_PPAT_WC			(1<<0)
151
-#define GEN8_PPAT_UC			(0<<0)
152
-#define GEN8_PPAT_ELLC_OVERRIDE		(0<<2)
153
-#define GEN8_PPAT(i, x)			((u64)(x) << ((i) * 8))
154
-
155
-#define GEN8_PPAT_GET_CA(x) ((x) & 3)
156
-#define GEN8_PPAT_GET_TC(x) ((x) & (3 << 2))
157
-#define GEN8_PPAT_GET_AGE(x) ((x) & (3 << 4))
158
-#define CHV_PPAT_GET_SNOOP(x) ((x) & (1 << 6))
159
-
160
-#define GEN8_PDE_IPS_64K BIT(11)
161
-#define GEN8_PDE_PS_2M   BIT(7)
162
-
163
-struct sg_table;
164
-
165
-struct intel_rotation_info {
166
-	struct intel_rotation_plane_info {
167
-		/* tiles */
168
-		unsigned int width, height, stride, offset;
169
-	} plane[2];
170
-} __packed;
171
-
172
-static inline void assert_intel_rotation_info_is_packed(void)
173
-{
174
-	BUILD_BUG_ON(sizeof(struct intel_rotation_info) != 8*sizeof(unsigned int));
175
-}
176
-
177
-struct intel_partial_info {
178
-	u64 offset;
179
-	unsigned int size;
180
-} __packed;
181
-
182
-static inline void assert_intel_partial_info_is_packed(void)
183
-{
184
-	BUILD_BUG_ON(sizeof(struct intel_partial_info) != sizeof(u64) + sizeof(unsigned int));
185
-}
186
-
187
-enum i915_ggtt_view_type {
188
-	I915_GGTT_VIEW_NORMAL = 0,
189
-	I915_GGTT_VIEW_ROTATED = sizeof(struct intel_rotation_info),
190
-	I915_GGTT_VIEW_PARTIAL = sizeof(struct intel_partial_info),
191
-};
192
-
193
-static inline void assert_i915_ggtt_view_type_is_unique(void)
194
-{
195
-	/* As we encode the size of each branch inside the union into its type,
196
-	 * we have to be careful that each branch has a unique size.
197
-	 */
198
-	switch ((enum i915_ggtt_view_type)0) {
199
-	case I915_GGTT_VIEW_NORMAL:
200
-	case I915_GGTT_VIEW_PARTIAL:
201
-	case I915_GGTT_VIEW_ROTATED:
202
-		/* gcc complains if these are identical cases */
203
-		break;
204
-	}
205
-}
206
-
207
-struct i915_ggtt_view {
208
-	enum i915_ggtt_view_type type;
209
-	union {
210
-		/* Members need to contain no holes/padding */
211
-		struct intel_partial_info partial;
212
-		struct intel_rotation_info rotated;
213
-	};
214
-};
215
-
216
-enum i915_cache_level;
217
-
218
-struct i915_vma;
219
-
220
-struct i915_page_dma {
221
-	struct page *page;
222
-	int order;
223
-	union {
224
-		dma_addr_t daddr;
225
-
226
-		/* For gen6/gen7 only. This is the offset in the GGTT
227
-		 * where the page directory entries for PPGTT begin
228
-		 */
229
-		u32 ggtt_offset;
230
-	};
231
-};
232
-
233
-#define px_base(px) (&(px)->base)
234
-#define px_page(px) (px_base(px)->page)
235
-#define px_dma(px) (px_base(px)->daddr)
236
-
237
-struct i915_page_table {
238
-	struct i915_page_dma base;
239
-	unsigned int used_ptes;
240
-};
241
-
242
-struct i915_page_directory {
243
-	struct i915_page_dma base;
244
-
245
-	struct i915_page_table *page_table[I915_PDES]; /* PDEs */
246
-	unsigned int used_pdes;
247
-};
248
-
249
-struct i915_page_directory_pointer {
250
-	struct i915_page_dma base;
251
-	struct i915_page_directory **page_directory;
252
-	unsigned int used_pdpes;
253
-};
254
-
255
-struct i915_pml4 {
256
-	struct i915_page_dma base;
257
-	struct i915_page_directory_pointer *pdps[GEN8_PML4ES_PER_PML4];
258
-};
259
-
260
-struct i915_vma_ops {
261
-	/* Map an object into an address space with the given cache flags. */
262
-	int (*bind_vma)(struct i915_vma *vma,
263
-			enum i915_cache_level cache_level,
264
-			u32 flags);
265
-	/*
266
-	 * Unmap an object from an address space. This usually consists of
267
-	 * setting the valid PTE entries to a reserved scratch page.
268
-	 */
269
-	void (*unbind_vma)(struct i915_vma *vma);
270
-
271
-	int (*set_pages)(struct i915_vma *vma);
272
-	void (*clear_pages)(struct i915_vma *vma);
273
-};
274
-
275
-struct pagestash {
276
-	spinlock_t lock;
277
-	struct pagevec pvec;
278
-};
279
-
280
-struct i915_address_space {
281
-	struct drm_mm mm;
282
-	struct drm_i915_private *i915;
283
-	struct device *dma;
284
-	/* Every address space belongs to a struct file - except for the global
285
-	 * GTT that is owned by the driver (and so @file is set to NULL). In
286
-	 * principle, no information should leak from one context to another
287
-	 * (or between files/processes etc) unless explicitly shared by the
288
-	 * owner. Tracking the owner is important in order to free up per-file
289
-	 * objects along with the file, to aide resource tracking, and to
290
-	 * assign blame.
291
-	 */
292
-	struct drm_i915_file_private *file;
293
-	u64 total;		/* size addr space maps (ex. 2GB for ggtt) */
294
-	u64 reserved;		/* size addr space reserved */
295
-
296
-	bool closed;
297
-
298
-	struct mutex mutex; /* protects vma and our lists */
299
-
300
-	struct i915_page_dma scratch_page;
301
-	struct i915_page_table *scratch_pt;
302
-	struct i915_page_directory *scratch_pd;
303
-	struct i915_page_directory_pointer *scratch_pdp; /* GEN8+ & 48b PPGTT */
304
-
305
-	/**
306
-	 * List of objects currently involved in rendering.
307
-	 *
308
-	 * Includes buffers having the contents of their GPU caches
309
-	 * flushed, not necessarily primitives. last_read_req
310
-	 * represents when the rendering involved will be completed.
311
-	 *
312
-	 * A reference is held on the buffer while on this list.
313
-	 */
314
-	struct list_head active_list;
315
-
316
-	/**
317
-	 * LRU list of objects which are not in the ringbuffer and
318
-	 * are ready to unbind, but are still in the GTT.
319
-	 *
320
-	 * last_read_req is NULL while an object is in this list.
321
-	 *
322
-	 * A reference is not held on the buffer while on this list,
323
-	 * as merely being GTT-bound shouldn't prevent its being
324
-	 * freed, and we'll pull it off the list in the free path.
325
-	 */
326
-	struct list_head inactive_list;
327
-
328
-	/**
329
-	 * List of vma that have been unbound.
330
-	 *
331
-	 * A reference is not held on the buffer while on this list.
332
-	 */
333
-	struct list_head unbound_list;
334
-
335
-	struct pagestash free_pages;
336
-
337
-	/* Some systems require uncached updates of the page directories */
338
-	bool pt_kmap_wc:1;
339
-
340
-	/* Some systems support read-only mappings for GGTT and/or PPGTT */
341
-	bool has_read_only:1;
342
-
343
-	/* FIXME: Need a more generic return type */
344
-	gen6_pte_t (*pte_encode)(dma_addr_t addr,
345
-				 enum i915_cache_level level,
346
-				 u32 flags); /* Create a valid PTE */
347
-	/* flags for pte_encode */
348
-#define PTE_READ_ONLY	(1<<0)
349
-	int (*allocate_va_range)(struct i915_address_space *vm,
350
-				 u64 start, u64 length);
351
-	void (*clear_range)(struct i915_address_space *vm,
352
-			    u64 start, u64 length);
353
-	void (*insert_page)(struct i915_address_space *vm,
354
-			    dma_addr_t addr,
355
-			    u64 offset,
356
-			    enum i915_cache_level cache_level,
357
-			    u32 flags);
358
-	void (*insert_entries)(struct i915_address_space *vm,
359
-			       struct i915_vma *vma,
360
-			       enum i915_cache_level cache_level,
361
-			       u32 flags);
362
-	void (*cleanup)(struct i915_address_space *vm);
363
-
364
-	struct i915_vma_ops vma_ops;
365
-
366
-	I915_SELFTEST_DECLARE(struct fault_attr fault_attr);
367
-	I915_SELFTEST_DECLARE(bool scrub_64K);
368
-};
369
-
370
-#define i915_is_ggtt(V) (!(V)->file)
371
-
372
-static inline bool
373
-i915_vm_is_48bit(const struct i915_address_space *vm)
374
-{
375
-	return (vm->total - 1) >> 32;
376
-}
377
-
378
-static inline bool
379
-i915_vm_has_scratch_64K(struct i915_address_space *vm)
380
-{
381
-	return vm->scratch_page.order == get_order(I915_GTT_PAGE_SIZE_64K);
382
-}
383
-
384
-/* The Graphics Translation Table is the way in which GEN hardware translates a
385
- * Graphics Virtual Address into a Physical Address. In addition to the normal
386
- * collateral associated with any va->pa translations GEN hardware also has a
387
- * portion of the GTT which can be mapped by the CPU and remain both coherent
388
- * and correct (in cases like swizzling). That region is referred to as GMADR in
389
- * the spec.
390
- */
391
-struct i915_ggtt {
392
-	struct i915_address_space vm;
393
-
394
-	struct io_mapping iomap;	/* Mapping to our CPU mappable region */
395
-	struct resource gmadr;          /* GMADR resource */
396
-	resource_size_t mappable_end;	/* End offset that we can CPU map */
397
-
398
-	/** "Graphics Stolen Memory" holds the global PTEs */
399
-	void __iomem *gsm;
400
-	void (*invalidate)(struct drm_i915_private *dev_priv);
401
-
402
-	bool do_idle_maps;
403
-
404
-	int mtrr;
405
-
406
-	struct drm_mm_node error_capture;
407
-};
408
-
409
-struct i915_hw_ppgtt {
410
-	struct i915_address_space vm;
411
-	struct kref ref;
412
-
413
-	unsigned long pd_dirty_rings;
414
-	union {
415
-		struct i915_pml4 pml4;		/* GEN8+ & 48b PPGTT */
416
-		struct i915_page_directory_pointer pdp;	/* GEN8+ */
417
-		struct i915_page_directory pd;		/* GEN6-7 */
418
-	};
419
-
420
-	void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m);
421
-};
422
-
423
-struct gen6_hw_ppgtt {
424
-	struct i915_hw_ppgtt base;
425
-
426
-	struct i915_vma *vma;
427
-	gen6_pte_t __iomem *pd_addr;
428
-	gen6_pte_t scratch_pte;
429
-
430
-	unsigned int pin_count;
431
-	bool scan_for_unused_pt;
432
-};
433
-
434
-#define __to_gen6_ppgtt(base) container_of(base, struct gen6_hw_ppgtt, base)
435
-
436
-static inline struct gen6_hw_ppgtt *to_gen6_ppgtt(struct i915_hw_ppgtt *base)
437
-{
438
-	BUILD_BUG_ON(offsetof(struct gen6_hw_ppgtt, base));
439
-	return __to_gen6_ppgtt(base);
440
-}
441
-
442
-/*
443
- * gen6_for_each_pde() iterates over every pde from start until start+length.
444
- * If start and start+length are not perfectly divisible, the macro will round
445
- * down and up as needed. Start=0 and length=2G effectively iterates over
446
- * every PDE in the system. The macro modifies ALL its parameters except 'pd',
447
- * so each of the other parameters should preferably be a simple variable, or
448
- * at most an lvalue with no side-effects!
449
- */
450
-#define gen6_for_each_pde(pt, pd, start, length, iter)			\
451
-	for (iter = gen6_pde_index(start);				\
452
-	     length > 0 && iter < I915_PDES &&				\
453
-		(pt = (pd)->page_table[iter], true);			\
454
-	     ({ u32 temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT);		\
455
-		    temp = min(temp - start, length);			\
456
-		    start += temp, length -= temp; }), ++iter)
457
-
458
-#define gen6_for_all_pdes(pt, pd, iter)					\
459
-	for (iter = 0;							\
460
-	     iter < I915_PDES &&					\
461
-		(pt = (pd)->page_table[iter], true);			\
462
-	     ++iter)
463
-
464
-static inline u32 i915_pte_index(u64 address, unsigned int pde_shift)
465
-{
466
-	const u32 mask = NUM_PTE(pde_shift) - 1;
467
-
468
-	return (address >> PAGE_SHIFT) & mask;
469
-}
470
-
471
-/* Helper to counts the number of PTEs within the given length. This count
472
- * does not cross a page table boundary, so the max value would be
473
- * GEN6_PTES for GEN6, and GEN8_PTES for GEN8.
474
-*/
475
-static inline u32 i915_pte_count(u64 addr, u64 length, unsigned int pde_shift)
476
-{
477
-	const u64 mask = ~((1ULL << pde_shift) - 1);
478
-	u64 end;
479
-
480
-	GEM_BUG_ON(length == 0);
481
-	GEM_BUG_ON(offset_in_page(addr | length));
482
-
483
-	end = addr + length;
484
-
485
-	if ((addr & mask) != (end & mask))
486
-		return NUM_PTE(pde_shift) - i915_pte_index(addr, pde_shift);
487
-
488
-	return i915_pte_index(end, pde_shift) - i915_pte_index(addr, pde_shift);
489
-}
490
-
491
-static inline u32 i915_pde_index(u64 addr, u32 shift)
492
-{
493
-	return (addr >> shift) & I915_PDE_MASK;
494
-}
495
-
496
-static inline u32 gen6_pte_index(u32 addr)
497
-{
498
-	return i915_pte_index(addr, GEN6_PDE_SHIFT);
499
-}
500
-
501
-static inline u32 gen6_pte_count(u32 addr, u32 length)
502
-{
503
-	return i915_pte_count(addr, length, GEN6_PDE_SHIFT);
504
-}
505
-
506
-static inline u32 gen6_pde_index(u32 addr)
507
-{
508
-	return i915_pde_index(addr, GEN6_PDE_SHIFT);
509
-}
510
-
511
-static inline unsigned int
512
-i915_pdpes_per_pdp(const struct i915_address_space *vm)
513
-{
514
-	if (i915_vm_is_48bit(vm))
515
-		return GEN8_PML4ES_PER_PML4;
516
-
517
-	return GEN8_3LVL_PDPES;
518
-}
519
-
520
-/* Equivalent to the gen6 version, For each pde iterates over every pde
521
- * between from start until start + length. On gen8+ it simply iterates
522
- * over every page directory entry in a page directory.
523
- */
524
-#define gen8_for_each_pde(pt, pd, start, length, iter)			\
525
-	for (iter = gen8_pde_index(start);				\
526
-	     length > 0 && iter < I915_PDES &&				\
527
-		(pt = (pd)->page_table[iter], true);			\
528
-	     ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT);		\
529
-		    temp = min(temp - start, length);			\
530
-		    start += temp, length -= temp; }), ++iter)
531
-
532
-#define gen8_for_each_pdpe(pd, pdp, start, length, iter)		\
533
-	for (iter = gen8_pdpe_index(start);				\
534
-	     length > 0 && iter < i915_pdpes_per_pdp(vm) &&		\
535
-		(pd = (pdp)->page_directory[iter], true);		\
536
-	     ({ u64 temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT);	\
537
-		    temp = min(temp - start, length);			\
538
-		    start += temp, length -= temp; }), ++iter)
539
-
540
-#define gen8_for_each_pml4e(pdp, pml4, start, length, iter)		\
541
-	for (iter = gen8_pml4e_index(start);				\
542
-	     length > 0 && iter < GEN8_PML4ES_PER_PML4 &&		\
543
-		(pdp = (pml4)->pdps[iter], true);			\
544
-	     ({ u64 temp = ALIGN(start+1, 1ULL << GEN8_PML4E_SHIFT);	\
545
-		    temp = min(temp - start, length);			\
546
-		    start += temp, length -= temp; }), ++iter)
547
-
548
-static inline u32 gen8_pte_index(u64 address)
549
-{
550
-	return i915_pte_index(address, GEN8_PDE_SHIFT);
551
-}
552
-
553
-static inline u32 gen8_pde_index(u64 address)
554
-{
555
-	return i915_pde_index(address, GEN8_PDE_SHIFT);
556
-}
557
-
558
-static inline u32 gen8_pdpe_index(u64 address)
559
-{
560
-	return (address >> GEN8_PDPE_SHIFT) & GEN8_PDPE_MASK;
561
-}
562
-
563
-static inline u32 gen8_pml4e_index(u64 address)
564
-{
565
-	return (address >> GEN8_PML4E_SHIFT) & GEN8_PML4E_MASK;
566
-}
567
-
568
-static inline u64 gen8_pte_count(u64 address, u64 length)
569
-{
570
-	return i915_pte_count(address, length, GEN8_PDE_SHIFT);
571
-}
572
-
573
-static inline dma_addr_t
574
-i915_page_dir_dma_addr(const struct i915_hw_ppgtt *ppgtt, const unsigned n)
575
-{
576
-	return px_dma(ppgtt->pdp.page_directory[n]);
577
-}
578
-
579
-static inline struct i915_ggtt *
580
-i915_vm_to_ggtt(struct i915_address_space *vm)
581
-{
582
-	GEM_BUG_ON(!i915_is_ggtt(vm));
583
-	return container_of(vm, struct i915_ggtt, vm);
584
-}
585
-
586
-#define INTEL_MAX_PPAT_ENTRIES 8
587
-#define INTEL_PPAT_PERFECT_MATCH (~0U)
588
-
589
-struct intel_ppat;
590
-
591
-struct intel_ppat_entry {
592
-	struct intel_ppat *ppat;
593
-	struct kref ref;
594
-	u8 value;
595
-};
596
-
597
-struct intel_ppat {
598
-	struct intel_ppat_entry entries[INTEL_MAX_PPAT_ENTRIES];
599
-	DECLARE_BITMAP(used, INTEL_MAX_PPAT_ENTRIES);
600
-	DECLARE_BITMAP(dirty, INTEL_MAX_PPAT_ENTRIES);
601
-	unsigned int max_entries;
602
-	u8 clear_value;
603
-	/*
604
-	 * Return a score to show how two PPAT values match,
605
-	 * a INTEL_PPAT_PERFECT_MATCH indicates a perfect match
606
-	 */
607
-	unsigned int (*match)(u8 src, u8 dst);
608
-	void (*update_hw)(struct drm_i915_private *i915);
609
-
610
-	struct drm_i915_private *i915;
611
-};
612
-
613
-const struct intel_ppat_entry *
614
-intel_ppat_get(struct drm_i915_private *i915, u8 value);
615
-void intel_ppat_put(const struct intel_ppat_entry *entry);
616
-
617
-int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915);
618
-void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915);
619
-
620
-int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv);
621
-int i915_ggtt_init_hw(struct drm_i915_private *dev_priv);
622
-int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv);
623
-void i915_ggtt_enable_guc(struct drm_i915_private *i915);
624
-void i915_ggtt_disable_guc(struct drm_i915_private *i915);
625
-int i915_gem_init_ggtt(struct drm_i915_private *dev_priv);
626
-void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv);
627
-
628
-int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv);
629
-void i915_ppgtt_release(struct kref *kref);
630
-struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv,
631
-					struct drm_i915_file_private *fpriv);
632
-void i915_ppgtt_close(struct i915_address_space *vm);
633
-static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
634
-{
635
-	if (ppgtt)
636
-		kref_get(&ppgtt->ref);
637
-}
638
-static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt)
639
-{
640
-	if (ppgtt)
641
-		kref_put(&ppgtt->ref, i915_ppgtt_release);
642
-}
643
-
644
-int gen6_ppgtt_pin(struct i915_hw_ppgtt *base);
645
-void gen6_ppgtt_unpin(struct i915_hw_ppgtt *base);
646
-
647
-void i915_check_and_clear_faults(struct drm_i915_private *dev_priv);
648
-void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv);
649
-void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv);
17
+struct drm_i915_gem_object;
18
+struct i915_address_space;
65019 
65120 int __must_check i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
65221 					    struct sg_table *pages);
....@@ -664,20 +33,18 @@
66433 			u64 start, u64 end, unsigned int flags);
66534 
66635 /* Flags used by pin/bind&friends. */
667
-#define PIN_NONBLOCK		BIT_ULL(0)
668
-#define PIN_MAPPABLE		BIT_ULL(1)
669
-#define PIN_ZONE_4G		BIT_ULL(2)
670
-#define PIN_NONFAULT		BIT_ULL(3)
671
-#define PIN_NOEVICT		BIT_ULL(4)
36
+#define PIN_NOEVICT		BIT_ULL(0)
37
+#define PIN_NOSEARCH		BIT_ULL(1)
38
+#define PIN_NONBLOCK		BIT_ULL(2)
39
+#define PIN_MAPPABLE		BIT_ULL(3)
40
+#define PIN_ZONE_4G		BIT_ULL(4)
41
+#define PIN_HIGH		BIT_ULL(5)
42
+#define PIN_OFFSET_BIAS		BIT_ULL(6)
43
+#define PIN_OFFSET_FIXED	BIT_ULL(7)
67244 
673
-#define PIN_MBZ			BIT_ULL(5) /* I915_VMA_PIN_OVERFLOW */
674
-#define PIN_GLOBAL		BIT_ULL(6) /* I915_VMA_GLOBAL_BIND */
675
-#define PIN_USER		BIT_ULL(7) /* I915_VMA_LOCAL_BIND */
676
-#define PIN_UPDATE		BIT_ULL(8)
45
+#define PIN_GLOBAL		BIT_ULL(10) /* I915_VMA_GLOBAL_BIND */
46
+#define PIN_USER		BIT_ULL(11) /* I915_VMA_LOCAL_BIND */
67747 
678
-#define PIN_HIGH		BIT_ULL(9)
679
-#define PIN_OFFSET_BIAS		BIT_ULL(10)
680
-#define PIN_OFFSET_FIXED	BIT_ULL(11)
681
-#define PIN_OFFSET_MASK		(-I915_GTT_PAGE_SIZE)
48
+#define PIN_OFFSET_MASK		I915_GTT_PAGE_MASK
68249 
68350 #endif