~hc/RK356X_SDK_RELEASE.git

..	..	@@ -28,10 +28,21 @@
28	28	#include <linux/spinlock.h>
29	29	#include <linux/notifier.h>
30	30	#include <linux/hrtimer.h>
	31	+#include <linux/io-64-nonatomic-lo-hi.h>
31	32
32	33	#include "i915_reg.h"
33	34
34	35	struct drm_i915_private;
	36	+struct intel_runtime_pm;
	37	+struct intel_uncore;
	38	+struct intel_gt;
	39	+
	40	+struct intel_uncore_mmio_debug {
	41	+ spinlock_t lock; /** lock is also taken in irq contexts. */
	42	+ int unclaimed_mmio_check;
	43	+ int saved_mmio_check;
	44	+ u32 suspend_count;
	45	+};
35	46
36	47	enum forcewake_domain_id {
37	48	FW_DOMAIN_ID_RENDER = 0,
..	..	@@ -62,25 +73,30 @@
62	73	};
63	74
64	75	struct intel_uncore_funcs {
65		- void (force_wake_get)(struct drm_i915_private dev_priv,
	76	+ void (force_wake_get)(struct intel_uncore uncore,
66	77	enum forcewake_domains domains);
67		- void (force_wake_put)(struct drm_i915_private dev_priv,
	78	+ void (force_wake_put)(struct intel_uncore uncore,
68	79	enum forcewake_domains domains);
69	80
70		- u8 (mmio_readb)(struct drm_i915_private dev_priv,
	81	+ enum forcewake_domains (read_fw_domains)(struct intel_uncore uncore,
	82	+ i915_reg_t r);
	83	+ enum forcewake_domains (write_fw_domains)(struct intel_uncore uncore,
	84	+ i915_reg_t r);
	85	+
	86	+ u8 (mmio_readb)(struct intel_uncore uncore,
71	87	i915_reg_t r, bool trace);
72		- u16 (mmio_readw)(struct drm_i915_private dev_priv,
	88	+ u16 (mmio_readw)(struct intel_uncore uncore,
73	89	i915_reg_t r, bool trace);
74		- u32 (mmio_readl)(struct drm_i915_private dev_priv,
	90	+ u32 (mmio_readl)(struct intel_uncore uncore,
75	91	i915_reg_t r, bool trace);
76		- u64 (mmio_readq)(struct drm_i915_private dev_priv,
	92	+ u64 (mmio_readq)(struct intel_uncore uncore,
77	93	i915_reg_t r, bool trace);
78	94
79		- void (mmio_writeb)(struct drm_i915_private dev_priv,
	95	+ void (mmio_writeb)(struct intel_uncore uncore,
80	96	i915_reg_t r, u8 val, bool trace);
81		- void (mmio_writew)(struct drm_i915_private dev_priv,
	97	+ void (mmio_writew)(struct intel_uncore uncore,
82	98	i915_reg_t r, u16 val, bool trace);
83		- void (mmio_writel)(struct drm_i915_private dev_priv,
	99	+ void (mmio_writel)(struct intel_uncore uncore,
84	100	i915_reg_t r, u32 val, bool trace);
85	101	};
86	102
..	..	@@ -92,7 +108,18 @@
92	108	};
93	109
94	110	struct intel_uncore {
	111	+ void __iomem *regs;
	112	+
	113	+ struct drm_i915_private *i915;
	114	+ struct intel_runtime_pm *rpm;
	115	+
95	116	spinlock_t lock; /** lock is also taken in irq contexts. */
	117	+
	118	+ unsigned int flags;
	119	+#define UNCORE_HAS_FORCEWAKE BIT(0)
	120	+#define UNCORE_HAS_FPGA_DBG_UNCLAIMED BIT(1)
	121	+#define UNCORE_HAS_DBG_UNCLAIMED BIT(2)
	122	+#define UNCORE_HAS_FIFO BIT(3)
96	123
97	124	const struct intel_forcewake_range *fw_domains_table;
98	125	unsigned int fw_domains_table_entries;
..	..	@@ -104,113 +131,292 @@
104	131
105	132	enum forcewake_domains fw_domains;
106	133	enum forcewake_domains fw_domains_active;
	134	+ enum forcewake_domains fw_domains_timer;
107	135	enum forcewake_domains fw_domains_saved; /* user domains saved for S3 */
108	136
109		- u32 fw_set;
110		- u32 fw_clear;
111		- u32 fw_reset;
112		-
113	137	struct intel_uncore_forcewake_domain {
	138	+ struct intel_uncore *uncore;
114	139	enum forcewake_domain_id id;
115	140	enum forcewake_domains mask;
116	141	unsigned int wake_count;
117	142	bool active;
118	143	struct hrtimer timer;
119		- i915_reg_t reg_set;
120		- i915_reg_t reg_ack;
121		- } fw_domain[FW_DOMAIN_ID_COUNT];
	144	+ u32 __iomem *reg_set;
	145	+ u32 __iomem *reg_ack;
	146	+ } *fw_domain[FW_DOMAIN_ID_COUNT];
122	147
123		- struct {
124		- unsigned int count;
	148	+ unsigned int user_forcewake_count;
125	149
126		- int saved_mmio_check;
127		- int saved_mmio_debug;
128		- } user_forcewake;
129		-
130		- int unclaimed_mmio_check;
	150	+ struct intel_uncore_mmio_debug *debug;
131	151	};
132	152
133	153	/* Iterate over initialised fw domains */
134		-#define for_each_fw_domain_masked(domain__, mask__, dev_priv__, tmp__) \
135		- for (tmp__ = (mask__); \
136		- tmp__ ? (domain__ = &(dev_priv__)->uncore.fw_domain[__mask_next_bit(tmp__)]), 1 : 0;)
	154	+#define for_each_fw_domain_masked(domain__, mask__, uncore__, tmp__) \
	155	+ for (tmp__ = (mask__); tmp__ ;) \
	156	+ for_each_if(domain__ = (uncore__)->fw_domain[__mask_next_bit(tmp__)])
137	157
138		-#define for_each_fw_domain(domain__, dev_priv__, tmp__) \
139		- for_each_fw_domain_masked(domain__, (dev_priv__)->uncore.fw_domains, dev_priv__, tmp__)
	158	+#define for_each_fw_domain(domain__, uncore__, tmp__) \
	159	+ for_each_fw_domain_masked(domain__, (uncore__)->fw_domains, uncore__, tmp__)
140	160
	161	+static inline bool
	162	+intel_uncore_has_forcewake(const struct intel_uncore *uncore)
	163	+{
	164	+ return uncore->flags & UNCORE_HAS_FORCEWAKE;
	165	+}
141	166
142		-void intel_uncore_sanitize(struct drm_i915_private *dev_priv);
143		-void intel_uncore_init(struct drm_i915_private *dev_priv);
144		-void intel_uncore_prune(struct drm_i915_private *dev_priv);
145		-bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv);
146		-bool intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv);
147		-void intel_uncore_fini(struct drm_i915_private *dev_priv);
148		-void intel_uncore_suspend(struct drm_i915_private *dev_priv);
149		-void intel_uncore_resume_early(struct drm_i915_private *dev_priv);
150		-void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv);
	167	+static inline bool
	168	+intel_uncore_has_fpga_dbg_unclaimed(const struct intel_uncore *uncore)
	169	+{
	170	+ return uncore->flags & UNCORE_HAS_FPGA_DBG_UNCLAIMED;
	171	+}
151	172
152		-u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv);
153		-void assert_forcewakes_inactive(struct drm_i915_private *dev_priv);
154		-void assert_forcewakes_active(struct drm_i915_private *dev_priv,
	173	+static inline bool
	174	+intel_uncore_has_dbg_unclaimed(const struct intel_uncore *uncore)
	175	+{
	176	+ return uncore->flags & UNCORE_HAS_DBG_UNCLAIMED;
	177	+}
	178	+
	179	+static inline bool
	180	+intel_uncore_has_fifo(const struct intel_uncore *uncore)
	181	+{
	182	+ return uncore->flags & UNCORE_HAS_FIFO;
	183	+}
	184	+
	185	+void
	186	+intel_uncore_mmio_debug_init_early(struct intel_uncore_mmio_debug *mmio_debug);
	187	+void intel_uncore_init_early(struct intel_uncore *uncore,
	188	+ struct drm_i915_private *i915);
	189	+int intel_uncore_init_mmio(struct intel_uncore *uncore);
	190	+void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore,
	191	+ struct intel_gt *gt);
	192	+bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore);
	193	+bool intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore);
	194	+void intel_uncore_fini_mmio(struct intel_uncore *uncore);
	195	+void intel_uncore_suspend(struct intel_uncore *uncore);
	196	+void intel_uncore_resume_early(struct intel_uncore *uncore);
	197	+void intel_uncore_runtime_resume(struct intel_uncore *uncore);
	198	+
	199	+void assert_forcewakes_inactive(struct intel_uncore *uncore);
	200	+void assert_forcewakes_active(struct intel_uncore *uncore,
155	201	enum forcewake_domains fw_domains);
156	202	const char *intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id);
157	203
158	204	enum forcewake_domains
159		-intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
	205	+intel_uncore_forcewake_for_reg(struct intel_uncore *uncore,
160	206	i915_reg_t reg, unsigned int op);
161	207	#define FW_REG_READ (1)
162	208	#define FW_REG_WRITE (2)
163	209
164		-void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
	210	+void intel_uncore_forcewake_get(struct intel_uncore *uncore,
165	211	enum forcewake_domains domains);
166		-void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
	212	+void intel_uncore_forcewake_put(struct intel_uncore *uncore,
167	213	enum forcewake_domains domains);
168		-/* Like above but the caller must manage the uncore.lock itself.
	214	+void intel_uncore_forcewake_put_delayed(struct intel_uncore *uncore,
	215	+ enum forcewake_domains domains);
	216	+void intel_uncore_forcewake_flush(struct intel_uncore *uncore,
	217	+ enum forcewake_domains fw_domains);
	218	+
	219	+/*
	220	+ * Like above but the caller must manage the uncore.lock itself.
169	221	* Must be used with I915_READ_FW and friends.
170	222	*/
171		-void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
	223	+void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore,
172	224	enum forcewake_domains domains);
173		-void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
	225	+void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore,
174	226	enum forcewake_domains domains);
175	227
176		-void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv);
177		-void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv);
	228	+void intel_uncore_forcewake_user_get(struct intel_uncore *uncore);
	229	+void intel_uncore_forcewake_user_put(struct intel_uncore *uncore);
178	230
179		-int __intel_wait_for_register(struct drm_i915_private *dev_priv,
	231	+int __intel_wait_for_register(struct intel_uncore *uncore,
180	232	i915_reg_t reg,
181	233	u32 mask,
182	234	u32 value,
183	235	unsigned int fast_timeout_us,
184	236	unsigned int slow_timeout_ms,
185	237	u32 *out_value);
186		-static inline
187		-int intel_wait_for_register(struct drm_i915_private *dev_priv,
188		- i915_reg_t reg,
189		- u32 mask,
190		- u32 value,
191		- unsigned int timeout_ms)
	238	+static inline int
	239	+intel_wait_for_register(struct intel_uncore *uncore,
	240	+ i915_reg_t reg,
	241	+ u32 mask,
	242	+ u32 value,
	243	+ unsigned int timeout_ms)
192	244	{
193		- return __intel_wait_for_register(dev_priv, reg, mask, value, 2,
	245	+ return __intel_wait_for_register(uncore, reg, mask, value, 2,
194	246	timeout_ms, NULL);
195	247	}
196		-int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
	248	+
	249	+int __intel_wait_for_register_fw(struct intel_uncore *uncore,
197	250	i915_reg_t reg,
198	251	u32 mask,
199	252	u32 value,
200	253	unsigned int fast_timeout_us,
201	254	unsigned int slow_timeout_ms,
202	255	u32 *out_value);
203		-static inline
204		-int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
205		- i915_reg_t reg,
206		- u32 mask,
207		- u32 value,
	256	+static inline int
	257	+intel_wait_for_register_fw(struct intel_uncore *uncore,
	258	+ i915_reg_t reg,
	259	+ u32 mask,
	260	+ u32 value,
208	261	unsigned int timeout_ms)
209	262	{
210		- return __intel_wait_for_register_fw(dev_priv, reg, mask, value,
	263	+ return __intel_wait_for_register_fw(uncore, reg, mask, value,
211	264	2, timeout_ms, NULL);
212	265	}
213	266
	267	+/* register access functions */
	268	+#define __raw_read(x__, s__) \
	269	+static inline u##x__ __raw_uncore_read##x__(const struct intel_uncore *uncore, \
	270	+ i915_reg_t reg) \
	271	+{ \
	272	+ return read##s__(uncore->regs + i915_mmio_reg_offset(reg)); \
	273	+}
	274	+
	275	+#define __raw_write(x__, s__) \
	276	+static inline void __raw_uncore_write##x__(const struct intel_uncore *uncore, \
	277	+ i915_reg_t reg, u##x__ val) \
	278	+{ \
	279	+ write##s__(val, uncore->regs + i915_mmio_reg_offset(reg)); \
	280	+}
	281	+__raw_read(8, b)
	282	+__raw_read(16, w)
	283	+__raw_read(32, l)
	284	+__raw_read(64, q)
	285	+
	286	+__raw_write(8, b)
	287	+__raw_write(16, w)
	288	+__raw_write(32, l)
	289	+__raw_write(64, q)
	290	+
	291	+#undef __raw_read
	292	+#undef __raw_write
	293	+
	294	+#define __uncore_read(name__, x__, s__, trace__) \
	295	+static inline u##x__ intel_uncore_##name__(struct intel_uncore *uncore, \
	296	+ i915_reg_t reg) \
	297	+{ \
	298	+ return uncore->funcs.mmio_read##s__(uncore, reg, (trace__)); \
	299	+}
	300	+
	301	+#define __uncore_write(name__, x__, s__, trace__) \
	302	+static inline void intel_uncore_##name__(struct intel_uncore *uncore, \
	303	+ i915_reg_t reg, u##x__ val) \
	304	+{ \
	305	+ uncore->funcs.mmio_write##s__(uncore, reg, val, (trace__)); \
	306	+}
	307	+
	308	+__uncore_read(read8, 8, b, true)
	309	+__uncore_read(read16, 16, w, true)
	310	+__uncore_read(read, 32, l, true)
	311	+__uncore_read(read16_notrace, 16, w, false)
	312	+__uncore_read(read_notrace, 32, l, false)
	313	+
	314	+__uncore_write(write8, 8, b, true)
	315	+__uncore_write(write16, 16, w, true)
	316	+__uncore_write(write, 32, l, true)
	317	+__uncore_write(write_notrace, 32, l, false)
	318	+
	319	+/* Be very careful with read/write 64-bit values. On 32-bit machines, they
	320	+ * will be implemented using 2 32-bit writes in an arbitrary order with
	321	+ * an arbitrary delay between them. This can cause the hardware to
	322	+ * act upon the intermediate value, possibly leading to corruption and
	323	+ * machine death. For this reason we do not support I915_WRITE64, or
	324	+ * uncore->funcs.mmio_writeq.
	325	+ *
	326	+ * When reading a 64-bit value as two 32-bit values, the delay may cause
	327	+ * the two reads to mismatch, e.g. a timestamp overflowing. Also note that
	328	+ * occasionally a 64-bit register does not actually support a full readq
	329	+ * and must be read using two 32-bit reads.
	330	+ *
	331	+ * You have been warned.
	332	+ */
	333	+__uncore_read(read64, 64, q, true)
	334	+
	335	+static inline u64
	336	+intel_uncore_read64_2x32(struct intel_uncore *uncore,
	337	+ i915_reg_t lower_reg, i915_reg_t upper_reg)
	338	+{
	339	+ u32 upper, lower, old_upper, loop = 0;
	340	+ upper = intel_uncore_read(uncore, upper_reg);
	341	+ do {
	342	+ old_upper = upper;
	343	+ lower = intel_uncore_read(uncore, lower_reg);
	344	+ upper = intel_uncore_read(uncore, upper_reg);
	345	+ } while (upper != old_upper && loop++ < 2);
	346	+ return (u64)upper << 32 \| lower;
	347	+}
	348	+
	349	+#define intel_uncore_posting_read(...) ((void)intel_uncore_read_notrace(__VA_ARGS__))
	350	+#define intel_uncore_posting_read16(...) ((void)intel_uncore_read16_notrace(__VA_ARGS__))
	351	+
	352	+#undef __uncore_read
	353	+#undef __uncore_write
	354	+
	355	+/* These are untraced mmio-accessors that are only valid to be used inside
	356	+ * critical sections, such as inside IRQ handlers, where forcewake is explicitly
	357	+ * controlled.
	358	+ *
	359	+ * Think twice, and think again, before using these.
	360	+ *
	361	+ * As an example, these accessors can possibly be used between:
	362	+ *
	363	+ * spin_lock_irq(&uncore->lock);
	364	+ * intel_uncore_forcewake_get__locked();
	365	+ *
	366	+ * and
	367	+ *
	368	+ * intel_uncore_forcewake_put__locked();
	369	+ * spin_unlock_irq(&uncore->lock);
	370	+ *
	371	+ *
	372	+ * Note: some registers may not need forcewake held, so
	373	+ * intel_uncore_forcewake_{get,put} can be omitted, see
	374	+ * intel_uncore_forcewake_for_reg().
	375	+ *
	376	+ * Certain architectures will die if the same cacheline is concurrently accessed
	377	+ * by different clients (e.g. on Ivybridge). Access to registers should
	378	+ * therefore generally be serialised, by either the dev_priv->uncore.lock or
	379	+ * a more localised lock guarding all access to that bank of registers.
	380	+ */
	381	+#define intel_uncore_read_fw(...) __raw_uncore_read32(__VA_ARGS__)
	382	+#define intel_uncore_write_fw(...) __raw_uncore_write32(__VA_ARGS__)
	383	+#define intel_uncore_write64_fw(...) __raw_uncore_write64(__VA_ARGS__)
	384	+#define intel_uncore_posting_read_fw(...) ((void)intel_uncore_read_fw(__VA_ARGS__))
	385	+
	386	+static inline void intel_uncore_rmw(struct intel_uncore *uncore,
	387	+ i915_reg_t reg, u32 clear, u32 set)
	388	+{
	389	+ u32 old, val;
	390	+
	391	+ old = intel_uncore_read(uncore, reg);
	392	+ val = (old & ~clear) \| set;
	393	+ if (val != old)
	394	+ intel_uncore_write(uncore, reg, val);
	395	+}
	396	+
	397	+static inline void intel_uncore_rmw_fw(struct intel_uncore *uncore,
	398	+ i915_reg_t reg, u32 clear, u32 set)
	399	+{
	400	+ u32 old, val;
	401	+
	402	+ old = intel_uncore_read_fw(uncore, reg);
	403	+ val = (old & ~clear) \| set;
	404	+ if (val != old)
	405	+ intel_uncore_write_fw(uncore, reg, val);
	406	+}
	407	+
	408	+static inline int intel_uncore_write_and_verify(struct intel_uncore *uncore,
	409	+ i915_reg_t reg, u32 val,
	410	+ u32 mask, u32 expected_val)
	411	+{
	412	+ u32 reg_val;
	413	+
	414	+ intel_uncore_write(uncore, reg, val);
	415	+ reg_val = intel_uncore_read(uncore, reg);
	416	+
	417	+ return (reg_val & mask) != expected_val ? -EINVAL : 0;
	418	+}
	419	+
214	420	#define raw_reg_read(base, reg) \
215	421	readl(base + i915_mmio_reg_offset(reg))
216	422	#define raw_reg_write(base, reg, value) \