forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/drivers/soc/qcom/rpmh-rsc.c
....@@ -6,17 +6,21 @@
66 #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME
77 
88 #include <linux/atomic.h>
9
+#include <linux/cpu_pm.h>
910 #include <linux/delay.h>
1011 #include <linux/interrupt.h>
1112 #include <linux/io.h>
13
+#include <linux/iopoll.h>
1214 #include <linux/kernel.h>
1315 #include <linux/list.h>
16
+#include <linux/module.h>
1417 #include <linux/of.h>
1518 #include <linux/of_irq.h>
1619 #include <linux/of_platform.h>
1720 #include <linux/platform_device.h>
1821 #include <linux/slab.h>
1922 #include <linux/spinlock.h>
23
+#include <linux/wait.h>
2024 
2125 #include <soc/qcom/cmd-db.h>
2226 #include <soc/qcom/tcs.h>
....@@ -30,21 +34,41 @@
3034 #define RSC_DRV_TCS_OFFSET		672
3135 #define RSC_DRV_CMD_OFFSET		20
3236 
33
-/* DRV Configuration Information Register */
37
+/* DRV HW Solver Configuration Information Register */
38
+#define DRV_SOLVER_CONFIG		0x04
39
+#define DRV_HW_SOLVER_MASK		1
40
+#define DRV_HW_SOLVER_SHIFT		24
41
+
42
+/* DRV TCS Configuration Information Register */
3443 #define DRV_PRNT_CHLD_CONFIG		0x0C
3544 #define DRV_NUM_TCS_MASK		0x3F
3645 #define DRV_NUM_TCS_SHIFT		6
3746 #define DRV_NCPT_MASK			0x1F
3847 #define DRV_NCPT_SHIFT			27
3948 
40
-/* Register offsets */
49
+/* Offsets for common TCS Registers, one bit per TCS */
4150 #define RSC_DRV_IRQ_ENABLE		0x00
4251 #define RSC_DRV_IRQ_STATUS		0x04
43
-#define RSC_DRV_IRQ_CLEAR		0x08
44
-#define RSC_DRV_CMD_WAIT_FOR_CMPL	0x10
52
+#define RSC_DRV_IRQ_CLEAR		0x08	/* w/o; write 1 to clear */
53
+
54
+/*
55
+ * Offsets for per TCS Registers.
56
+ *
57
+ * TCSes start at 0x10 from tcs_base and are stored one after another.
58
+ * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one
59
+ * of the below to find a register.
60
+ */
61
+#define RSC_DRV_CMD_WAIT_FOR_CMPL	0x10	/* 1 bit per command */
4562 #define RSC_DRV_CONTROL			0x14
46
-#define RSC_DRV_STATUS			0x18
47
-#define RSC_DRV_CMD_ENABLE		0x1C
63
+#define RSC_DRV_STATUS			0x18	/* zero if tcs is busy */
64
+#define RSC_DRV_CMD_ENABLE		0x1C	/* 1 bit per command */
65
+
66
+/*
67
+ * Offsets for per command in a TCS.
68
+ *
69
+ * Commands (up to 16) start at 0x30 in a TCS; multiply command index
70
+ * by RSC_DRV_CMD_OFFSET and add one of the below to find a register.
71
+ */
4872 #define RSC_DRV_CMD_MSGID		0x30
4973 #define RSC_DRV_CMD_ADDR		0x34
5074 #define RSC_DRV_CMD_DATA		0x38
....@@ -61,90 +85,183 @@
6185 #define CMD_STATUS_ISSUED		BIT(8)
6286 #define CMD_STATUS_COMPL		BIT(16)
6387 
64
-static u32 read_tcs_reg(struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)
88
+/*
89
+ * Here's a high level overview of how all the registers in RPMH work
90
+ * together:
91
+ *
92
+ * - The main rpmh-rsc address is the base of a register space that can
93
+ *   be used to find overall configuration of the hardware
94
+ *   (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
95
+ *   space are all the TCS blocks. The offset of the TCS blocks is
96
+ *   specified in the device tree by "qcom,tcs-offset" and used to
97
+ *   compute tcs_base.
98
+ * - TCS blocks come one after another. Type, count, and order are
99
+ *   specified by the device tree as "qcom,tcs-config".
100
+ * - Each TCS block has some registers, then space for up to 16 commands.
101
+ *   Note that though address space is reserved for 16 commands, fewer
102
+ *   might be present. See ncpt (num cmds per TCS).
103
+ *
104
+ * Here's a picture:
105
+ *
106
+ *  +---------------------------------------------------+
107
+ *  |RSC                                                |
108
+ *  | ctrl                                              |
109
+ *  |                                                   |
110
+ *  | Drvs:                                             |
111
+ *  | +-----------------------------------------------+ |
112
+ *  | |DRV0                                           | |
113
+ *  | | ctrl/config                                   | |
114
+ *  | | IRQ                                           | |
115
+ *  | |                                               | |
116
+ *  | | TCSes:                                        | |
117
+ *  | | +------------------------------------------+  | |
118
+ *  | | |TCS0  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
119
+ *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
120
+ *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
121
+ *  | | +------------------------------------------+  | |
122
+ *  | | +------------------------------------------+  | |
123
+ *  | | |TCS1  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
124
+ *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
125
+ *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
126
+ *  | | +------------------------------------------+  | |
127
+ *  | | +------------------------------------------+  | |
128
+ *  | | |TCS2  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
129
+ *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
130
+ *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
131
+ *  | | +------------------------------------------+  | |
132
+ *  | |                    ......                     | |
133
+ *  | +-----------------------------------------------+ |
134
+ *  | +-----------------------------------------------+ |
135
+ *  | |DRV1                                           | |
136
+ *  | | (same as DRV0)                                | |
137
+ *  | +-----------------------------------------------+ |
138
+ *  |                      ......                       |
139
+ *  +---------------------------------------------------+
140
+ */
141
+
142
+static inline void __iomem *
143
+tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id)
65144 {
66
-	return readl_relaxed(drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id +
67
-			     RSC_DRV_CMD_OFFSET * cmd_id);
145
+	return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg;
68146 }
69147 
70
-static void write_tcs_cmd(struct rsc_drv *drv, int reg, int tcs_id, int cmd_id,
148
+static inline void __iomem *
149
+tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)
150
+{
151
+	return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id;
152
+}
153
+
154
+static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
155
+			int cmd_id)
156
+{
157
+	return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
158
+}
159
+
160
+static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id)
161
+{
162
+	return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id));
163
+}
164
+
165
+static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
166
+			  int cmd_id, u32 data)
167
+{
168
+	writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
169
+}
170
+
171
+static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id,
71172 			  u32 data)
72173 {
73
-	writel_relaxed(data, drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id +
74
-		       RSC_DRV_CMD_OFFSET * cmd_id);
174
+	writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id));
75175 }
76176 
77
-static void write_tcs_reg(struct rsc_drv *drv, int reg, int tcs_id, u32 data)
78
-{
79
-	writel_relaxed(data, drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id);
80
-}
81
-
82
-static void write_tcs_reg_sync(struct rsc_drv *drv, int reg, int tcs_id,
177
+static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id,
83178 			       u32 data)
84179 {
85
-	writel(data, drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id);
86
-	for (;;) {
87
-		if (data == readl(drv->tcs_base + reg +
88
-				  RSC_DRV_TCS_OFFSET * tcs_id))
89
-			break;
180
+	int i;
181
+
182
+	writel(data, tcs_reg_addr(drv, reg, tcs_id));
183
+
184
+	/*
185
+	 * Wait until we read back the same value.  Use a counter rather than
186
+	 * ktime for timeout since this may be called after timekeeping stops.
187
+	 */
188
+	for (i = 0; i < USEC_PER_SEC; i++) {
189
+		if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data)
190
+			return;
90191 		udelay(1);
91192 	}
193
+	pr_err("%s: error writing %#x to %d:%#x\n", drv->name,
194
+	       data, tcs_id, reg);
92195 }
93196 
197
+/**
198
+ * tcs_is_free() - Return if a TCS is totally free.
199
+ * @drv:    The RSC controller.
200
+ * @tcs_id: The global ID of this TCS.
201
+ *
202
+ * Returns true if nobody has claimed this TCS (by setting tcs_in_use).
203
+ *
204
+ * Context: Must be called with the drv->lock held.
205
+ *
206
+ * Return: true if the given TCS is free.
207
+ */
94208 static bool tcs_is_free(struct rsc_drv *drv, int tcs_id)
95209 {
96
-	return !test_bit(tcs_id, drv->tcs_in_use) &&
97
-	       read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id, 0);
210
+	return !test_bit(tcs_id, drv->tcs_in_use);
98211 }
99212 
100
-static struct tcs_group *get_tcs_of_type(struct rsc_drv *drv, int type)
101
-{
102
-	return &drv->tcs[type];
103
-}
104
-
105
-static int tcs_invalidate(struct rsc_drv *drv, int type)
213
+/**
214
+ * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
215
+ * @drv:  The RSC controller.
216
+ * @type: SLEEP_TCS or WAKE_TCS
217
+ *
218
+ * This will clear the "slots" variable of the given tcs_group and also
219
+ * tell the hardware to forget about all entries.
220
+ *
221
+ * The caller must ensure that no other RPMH actions are happening when this
222
+ * function is called, since otherwise the device may immediately become
223
+ * used again even before this function exits.
224
+ */
225
+static void tcs_invalidate(struct rsc_drv *drv, int type)
106226 {
107227 	int m;
108
-	struct tcs_group *tcs;
228
+	struct tcs_group *tcs = &drv->tcs[type];
109229 
110
-	tcs = get_tcs_of_type(drv, type);
111
-
112
-	spin_lock(&tcs->lock);
113
-	if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) {
114
-		spin_unlock(&tcs->lock);
115
-		return 0;
116
-	}
230
+	/* Caller ensures nobody else is running so no lock */
231
+	if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS))
232
+		return;
117233 
118234 	for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) {
119
-		if (!tcs_is_free(drv, m)) {
120
-			spin_unlock(&tcs->lock);
121
-			return -EAGAIN;
122
-		}
123235 		write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0);
124236 		write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, m, 0);
125237 	}
126238 	bitmap_zero(tcs->slots, MAX_TCS_SLOTS);
127
-	spin_unlock(&tcs->lock);
128
-
129
-	return 0;
130239 }
131240 
132241 /**
133
- * rpmh_rsc_invalidate - Invalidate sleep and wake TCSes
242
+ * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
243
+ * @drv: The RSC controller.
134244  *
135
- * @drv: the RSC controller
245
+ * The caller must ensure that no other RPMH actions are happening when this
246
+ * function is called, since otherwise the device may immediately become
247
+ * used again even before this function exits.
136248  */
137
-int rpmh_rsc_invalidate(struct rsc_drv *drv)
249
+void rpmh_rsc_invalidate(struct rsc_drv *drv)
138250 {
139
-	int ret;
140
-
141
-	ret = tcs_invalidate(drv, SLEEP_TCS);
142
-	if (!ret)
143
-		ret = tcs_invalidate(drv, WAKE_TCS);
144
-
145
-	return ret;
251
+	tcs_invalidate(drv, SLEEP_TCS);
252
+	tcs_invalidate(drv, WAKE_TCS);
146253 }
147254 
255
+/**
256
+ * get_tcs_for_msg() - Get the tcs_group used to send the given message.
257
+ * @drv: The RSC controller.
258
+ * @msg: The message we want to send.
259
+ *
260
+ * This is normally pretty straightforward except if we are trying to send
261
+ * an ACTIVE_ONLY message but don't have any active_only TCSes.
262
+ *
263
+ * Return: A pointer to a tcs_group or an ERR_PTR.
264
+ */
148265 static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,
149266 					 const struct tcs_request *msg)
150267 {
....@@ -168,15 +285,33 @@
168285 	/*
169286 	 * If we are making an active request on a RSC that does not have a
170287 	 * dedicated TCS for active state use, then re-purpose a wake TCS to
171
-	 * send active votes.
288
+	 * send active votes. This is safe because we ensure any active-only
289
+	 * transfers have finished before we use it (maybe by running from
290
+	 * the last CPU in PM code).
172291 	 */
173
-	tcs = get_tcs_of_type(drv, type);
292
+	tcs = &drv->tcs[type];
174293 	if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs)
175
-		tcs = get_tcs_of_type(drv, WAKE_TCS);
294
+		tcs = &drv->tcs[WAKE_TCS];
176295 
177296 	return tcs;
178297 }
179298 
299
+/**
300
+ * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
301
+ * @drv:    The RSC controller.
302
+ * @tcs_id: The global ID of this TCS.
303
+ *
304
+ * For ACTIVE_ONLY transfers we want to call back into the client when the
305
+ * transfer finishes. To do this we need the "request" that the client
306
+ * originally provided us. This function grabs the request that we stashed
307
+ * when we started the transfer.
308
+ *
309
+ * This only makes sense for ACTIVE_ONLY transfers since those are the only
310
+ * ones we track sending (the only ones we enable interrupts for and the only
311
+ * ones we call back to the client for).
312
+ *
313
+ * Return: The stashed request.
314
+ */
180315 static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,
181316 						  int tcs_id)
182317 {
....@@ -192,6 +327,23 @@
192327 	return NULL;
193328 }
194329 
330
+/**
331
+ * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
332
+ * @drv:     The controller.
333
+ * @tcs_id:  The global ID of this TCS.
334
+ * @trigger: If true then untrigger/retrigger. If false then just untrigger.
335
+ *
336
+ * In the normal case we only ever call with "trigger=true" to start a
337
+ * transfer. That will un-trigger/disable the TCS from the last transfer
338
+ * then trigger/enable for this transfer.
339
+ *
340
+ * If we borrowed a wake TCS for an active-only transfer we'll also call
341
+ * this function with "trigger=false" to just do the un-trigger/disable
342
+ * before using the TCS for wake purposes again.
343
+ *
344
+ * Note that the AP is only in charge of triggering active-only transfers.
345
+ * The AP never triggers sleep/wake values using this function.
346
+ */
195347 static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger)
196348 {
197349 	u32 enable;
....@@ -201,7 +353,7 @@
201353 	 * While clearing ensure that the AMC mode trigger is cleared
202354 	 * and then the mode enable is cleared.
203355 	 */
204
-	enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id, 0);
356
+	enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id);
205357 	enable &= ~TCS_AMC_MODE_TRIGGER;
206358 	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
207359 	enable &= ~TCS_AMC_MODE_ENABLE;
....@@ -216,20 +368,36 @@
216368 	}
217369 }
218370 
371
+/**
372
+ * enable_tcs_irq() - Enable or disable interrupts on the given TCS.
373
+ * @drv:     The controller.
374
+ * @tcs_id:  The global ID of this TCS.
375
+ * @enable:  If true then enable; if false then disable
376
+ *
377
+ * We only ever call this when we borrow a wake TCS for an active-only
378
+ * transfer. For active-only TCSes interrupts are always left enabled.
379
+ */
219380 static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable)
220381 {
221382 	u32 data;
222383 
223
-	data = read_tcs_reg(drv, RSC_DRV_IRQ_ENABLE, 0, 0);
384
+	data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE);
224385 	if (enable)
225386 		data |= BIT(tcs_id);
226387 	else
227388 		data &= ~BIT(tcs_id);
228
-	write_tcs_reg(drv, RSC_DRV_IRQ_ENABLE, 0, data);
389
+	writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE);
229390 }
230391 
231392 /**
232
- * tcs_tx_done: TX Done interrupt handler
393
+ * tcs_tx_done() - TX Done interrupt handler.
394
+ * @irq: The IRQ number (ignored).
395
+ * @p:   Pointer to "struct rsc_drv".
396
+ *
397
+ * Called for ACTIVE_ONLY transfers (those are the only ones we enable the
398
+ * IRQ for) when a transfer is done.
399
+ *
400
+ * Return: IRQ_HANDLED
233401  */
234402 static irqreturn_t tcs_tx_done(int irq, void *p)
235403 {
....@@ -239,7 +407,7 @@
239407 	const struct tcs_request *req;
240408 	struct tcs_cmd *cmd;
241409 
242
-	irq_status = read_tcs_reg(drv, RSC_DRV_IRQ_STATUS, 0, 0);
410
+	irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS);
243411 
244412 	for_each_set_bit(i, &irq_status, BITS_PER_LONG) {
245413 		req = get_req_from_tcs(drv, i);
....@@ -253,7 +421,7 @@
253421 			u32 sts;
254422 
255423 			cmd = &req->cmds[j];
256
-			sts = read_tcs_reg(drv, RSC_DRV_CMD_STATUS, i, j);
424
+			sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j);
257425 			if (!(sts & CMD_STATUS_ISSUED) ||
258426 			   ((req->wait_for_compl || cmd->wait) &&
259427 			   !(sts & CMD_STATUS_COMPL))) {
....@@ -276,7 +444,7 @@
276444 		/* Reclaim the TCS */
277445 		write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0);
278446 		write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, i, 0);
279
-		write_tcs_reg(drv, RSC_DRV_IRQ_CLEAR, 0, BIT(i));
447
+		writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR);
280448 		spin_lock(&drv->lock);
281449 		clear_bit(i, drv->tcs_in_use);
282450 		/*
....@@ -287,6 +455,7 @@
287455 		if (!drv->tcs[ACTIVE_TCS].num_tcs)
288456 			enable_tcs_irq(drv, i, false);
289457 		spin_unlock(&drv->lock);
458
+		wake_up(&drv->tcs_wait);
290459 		if (req)
291460 			rpmh_tx_done(req, err);
292461 	}
....@@ -294,6 +463,16 @@
294463 	return IRQ_HANDLED;
295464 }
296465 
466
+/**
467
+ * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
468
+ * @drv:    The controller.
469
+ * @tcs_id: The global ID of this TCS.
470
+ * @cmd_id: The index within the TCS to start writing.
471
+ * @msg:    The message we want to send, which will contain several addr/data
472
+ *          pairs to program (but few enough that they all fit in one TCS).
473
+ *
474
+ * This is used for all types of transfers (active, sleep, and wake).
475
+ */
297476 static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,
298477 			       const struct tcs_request *msg)
299478 {
....@@ -307,7 +486,7 @@
307486 	cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0;
308487 	cmd_msgid |= CMD_MSGID_WRITE;
309488 
310
-	cmd_complete = read_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0);
489
+	cmd_complete = read_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id);
311490 
312491 	for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) {
313492 		cmd = &msg->cmds[i];
....@@ -319,14 +498,34 @@
319498 		write_tcs_cmd(drv, RSC_DRV_CMD_MSGID, tcs_id, j, msgid);
320499 		write_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j, cmd->addr);
321500 		write_tcs_cmd(drv, RSC_DRV_CMD_DATA, tcs_id, j, cmd->data);
322
-		trace_rpmh_send_msg(drv, tcs_id, j, msgid, cmd);
501
+		// trace_rpmh_send_msg_rcuidle(drv, tcs_id, j, msgid, cmd);
323502 	}
324503 
325504 	write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, cmd_complete);
326
-	cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0);
505
+	cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id);
327506 	write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable);
328507 }
329508 
509
+/**
510
+ * check_for_req_inflight() - Look to see if conflicting cmds are in flight.
511
+ * @drv: The controller.
512
+ * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers.
513
+ * @msg: The message we want to send, which will contain several addr/data
514
+ *       pairs to program (but few enough that they all fit in one TCS).
515
+ *
516
+ * This will walk through the TCSes in the group and check if any of them
517
+ * appear to be sending to addresses referenced in the message. If it finds
518
+ * one it'll return -EBUSY.
519
+ *
520
+ * Only for use for active-only transfers.
521
+ *
522
+ * Must be called with the drv->lock held since that protects tcs_in_use.
523
+ *
524
+ * Return: 0 if nothing in flight or -EBUSY if we should try again later.
525
+ *         The caller must re-enable interrupts between tries since that's
526
+ *         the only way tcs_is_free() will ever return true and the only way
527
+ *         RSC_DRV_CMD_ENABLE will ever be cleared.
528
+ */
330529 static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,
331530 				  const struct tcs_request *msg)
332531 {
....@@ -339,10 +538,10 @@
339538 		if (tcs_is_free(drv, tcs_id))
340539 			continue;
341540 
342
-		curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0);
541
+		curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id);
343542 
344543 		for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) {
345
-			addr = read_tcs_reg(drv, RSC_DRV_CMD_ADDR, tcs_id, j);
544
+			addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j);
346545 			for (k = 0; k < msg->num_cmds; k++) {
347546 				if (addr == msg->cmds[k].addr)
348547 					return -EBUSY;
....@@ -353,6 +552,15 @@
353552 	return 0;
354553 }
355554 
555
+/**
556
+ * find_free_tcs() - Find free tcs in the given tcs_group; only for active.
557
+ * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
558
+ *       we borrowed it because there are zero active-only ones).
559
+ *
560
+ * Must be called with the drv->lock held since that protects tcs_in_use.
561
+ *
562
+ * Return: The first tcs that's free.
563
+ */
356564 static int find_free_tcs(struct tcs_group *tcs)
357565 {
358566 	int i;
....@@ -365,35 +573,75 @@
365573 	return -EBUSY;
366574 }
367575 
368
-static int tcs_write(struct rsc_drv *drv, const struct tcs_request *msg)
576
+/**
577
+ * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active.
578
+ * @drv: The controller.
579
+ * @tcs: The tcs_group used for ACTIVE_ONLY transfers.
580
+ * @msg: The data to be sent.
581
+ *
582
+ * Claims a tcs in the given tcs_group while making sure that no existing cmd
583
+ * is in flight that would conflict with the one in @msg.
584
+ *
585
+ * Context: Must be called with the drv->lock held since that protects
586
+ * tcs_in_use.
587
+ *
588
+ * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight
589
+ * or the tcs_group is full.
590
+ */
591
+static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs,
592
+			     const struct tcs_request *msg)
369593 {
370
-	struct tcs_group *tcs;
371
-	int tcs_id;
372
-	unsigned long flags;
373594 	int ret;
374595 
375
-	tcs = get_tcs_for_msg(drv, msg);
376
-	if (IS_ERR(tcs))
377
-		return PTR_ERR(tcs);
378
-
379
-	spin_lock_irqsave(&tcs->lock, flags);
380
-	spin_lock(&drv->lock);
381596 	/*
382597 	 * The h/w does not like if we send a request to the same address,
383598 	 * when one is already in-flight or being processed.
384599 	 */
385600 	ret = check_for_req_inflight(drv, tcs, msg);
386
-	if (ret) {
387
-		spin_unlock(&drv->lock);
388
-		goto done_write;
389
-	}
601
+	if (ret)
602
+		return ret;
390603 
391
-	tcs_id = find_free_tcs(tcs);
392
-	if (tcs_id < 0) {
393
-		ret = tcs_id;
394
-		spin_unlock(&drv->lock);
395
-		goto done_write;
396
-	}
604
+	return find_free_tcs(tcs);
605
+}
606
+
607
+/**
608
+ * rpmh_rsc_send_data() - Write / trigger active-only message.
609
+ * @drv: The controller.
610
+ * @msg: The data to be sent.
611
+ *
612
+ * NOTES:
613
+ * - This is only used for "ACTIVE_ONLY" since the limitations of this
614
+ *   function don't make sense for sleep/wake cases.
615
+ * - To do the transfer, we will grab a whole TCS for ourselves--we don't
616
+ *   try to share. If there are none available we'll wait indefinitely
617
+ *   for a free one.
618
+ * - This function will not wait for the commands to be finished, only for
619
+ *   data to be programmed into the RPMh. See rpmh_tx_done() which will
620
+ *   be called when the transfer is fully complete.
621
+ * - This function must be called with interrupts enabled. If the hardware
622
+ *   is busy doing someone else's transfer we need that transfer to fully
623
+ *   finish so that we can have the hardware, and to fully finish it needs
624
+ *   the interrupt handler to run. If the interrupts is set to run on the
625
+ *   active CPU this can never happen if interrupts are disabled.
626
+ *
627
+ * Return: 0 on success, -EINVAL on error.
628
+ */
629
+int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)
630
+{
631
+	struct tcs_group *tcs;
632
+	int tcs_id;
633
+	unsigned long flags;
634
+
635
+	tcs = get_tcs_for_msg(drv, msg);
636
+	if (IS_ERR(tcs))
637
+		return PTR_ERR(tcs);
638
+
639
+	spin_lock_irqsave(&drv->lock, flags);
640
+
641
+	/* Wait forever for a free tcs. It better be there eventually! */
642
+	wait_event_lock_irq(drv->tcs_wait,
643
+			    (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0,
644
+			    drv->lock);
397645 
398646 	tcs->req[tcs_id - tcs->offset] = msg;
399647 	set_bit(tcs_id, drv->tcs_in_use);
....@@ -407,98 +655,53 @@
407655 		write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0);
408656 		enable_tcs_irq(drv, tcs_id, true);
409657 	}
410
-	spin_unlock(&drv->lock);
658
+	spin_unlock_irqrestore(&drv->lock, flags);
411659 
660
+	/*
661
+	 * These two can be done after the lock is released because:
662
+	 * - We marked "tcs_in_use" under lock.
663
+	 * - Once "tcs_in_use" has been marked nobody else could be writing
664
+	 *   to these registers until the interrupt goes off.
665
+	 * - The interrupt can't go off until we trigger w/ the last line
666
+	 *   of __tcs_set_trigger() below.
667
+	 */
412668 	__tcs_buffer_write(drv, tcs_id, 0, msg);
413669 	__tcs_set_trigger(drv, tcs_id, true);
414670 
415
-done_write:
416
-	spin_unlock_irqrestore(&tcs->lock, flags);
417
-	return ret;
671
+	return 0;
418672 }
419673 
420674 /**
421
- * rpmh_rsc_send_data: Validate the incoming message and write to the
422
- * appropriate TCS block.
675
+ * find_slots() - Find a place to write the given message.
676
+ * @tcs:    The tcs group to search.
677
+ * @msg:    The message we want to find room for.
678
+ * @tcs_id: If we return 0 from the function, we return the global ID of the
679
+ *          TCS to write to here.
680
+ * @cmd_id: If we return 0 from the function, we return the index of
681
+ *          the command array of the returned TCS where the client should
682
+ *          start writing the message.
423683  *
424
- * @drv: the controller
425
- * @msg: the data to be sent
684
+ * Only for use on sleep/wake TCSes since those are the only ones we maintain
685
+ * tcs->slots for.
426686  *
427
- * Return: 0 on success, -EINVAL on error.
428
- * Note: This call blocks until a valid data is written to the TCS.
687
+ * Return: -ENOMEM if there was no room, else 0.
429688  */
430
-int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)
431
-{
432
-	int ret;
433
-
434
-	if (!msg || !msg->cmds || !msg->num_cmds ||
435
-	    msg->num_cmds > MAX_RPMH_PAYLOAD) {
436
-		WARN_ON(1);
437
-		return -EINVAL;
438
-	}
439
-
440
-	do {
441
-		ret = tcs_write(drv, msg);
442
-		if (ret == -EBUSY) {
443
-			pr_info_ratelimited("TCS Busy, retrying RPMH message send: addr=%#x\n",
444
-					    msg->cmds[0].addr);
445
-			udelay(10);
446
-		}
447
-	} while (ret == -EBUSY);
448
-
449
-	return ret;
450
-}
451
-
452
-static int find_match(const struct tcs_group *tcs, const struct tcs_cmd *cmd,
453
-		      int len)
454
-{
455
-	int i, j;
456
-
457
-	/* Check for already cached commands */
458
-	for_each_set_bit(i, tcs->slots, MAX_TCS_SLOTS) {
459
-		if (tcs->cmd_cache[i] != cmd[0].addr)
460
-			continue;
461
-		if (i + len >= tcs->num_tcs * tcs->ncpt)
462
-			goto seq_err;
463
-		for (j = 0; j < len; j++) {
464
-			if (tcs->cmd_cache[i + j] != cmd[j].addr)
465
-				goto seq_err;
466
-		}
467
-		return i;
468
-	}
469
-
470
-	return -ENODATA;
471
-
472
-seq_err:
473
-	WARN(1, "Message does not match previous sequence.\n");
474
-	return -EINVAL;
475
-}
476
-
477689 static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg,
478690 		      int *tcs_id, int *cmd_id)
479691 {
480692 	int slot, offset;
481693 	int i = 0;
482694 
483
-	/* Find if we already have the msg in our TCS */
484
-	slot = find_match(tcs, msg->cmds, msg->num_cmds);
485
-	if (slot >= 0)
486
-		goto copy_data;
487
-
488
-	/* Do over, until we can fit the full payload in a TCS */
695
+	/* Do over, until we can fit the full payload in a single TCS */
489696 	do {
490697 		slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS,
491698 						  i, msg->num_cmds, 0);
492
-		if (slot == tcs->num_tcs * tcs->ncpt)
699
+		if (slot >= tcs->num_tcs * tcs->ncpt)
493700 			return -ENOMEM;
494701 		i += tcs->ncpt;
495702 	} while (slot + msg->num_cmds - 1 >= i);
496703 
497
-copy_data:
498704 	bitmap_set(tcs->slots, slot, msg->num_cmds);
499
-	/* Copy the addresses of the resources over to the slots */
500
-	for (i = 0; i < msg->num_cmds; i++)
501
-		tcs->cmd_cache[slot + i] = msg->cmds[i].addr;
502705 
503706 	offset = slot / tcs->ncpt;
504707 	*tcs_id = offset + tcs->offset;
....@@ -507,52 +710,157 @@
507710 	return 0;
508711 }
509712 
510
-static int tcs_ctrl_write(struct rsc_drv *drv, const struct tcs_request *msg)
713
+/**
714
+ * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
715
+ * @drv: The controller.
716
+ * @msg: The data to be written to the controller.
717
+ *
718
+ * This should only be called for for sleep/wake state, never active-only
719
+ * state.
720
+ *
721
+ * The caller must ensure that no other RPMH actions are happening and the
722
+ * controller is idle when this function is called since it runs lockless.
723
+ *
724
+ * Return: 0 if no error; else -error.
725
+ */
726
+int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)
511727 {
512728 	struct tcs_group *tcs;
513729 	int tcs_id = 0, cmd_id = 0;
514
-	unsigned long flags;
515730 	int ret;
516731 
517732 	tcs = get_tcs_for_msg(drv, msg);
518733 	if (IS_ERR(tcs))
519734 		return PTR_ERR(tcs);
520735 
521
-	spin_lock_irqsave(&tcs->lock, flags);
522736 	/* find the TCS id and the command in the TCS to write to */
523737 	ret = find_slots(tcs, msg, &tcs_id, &cmd_id);
524738 	if (!ret)
525739 		__tcs_buffer_write(drv, tcs_id, cmd_id, msg);
526
-	spin_unlock_irqrestore(&tcs->lock, flags);
527740 
528741 	return ret;
529742 }
530743 
531744 /**
532
- * rpmh_rsc_write_ctrl_data: Write request to the controller
745
+ * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
746
+ * @drv: The controller
533747  *
534
- * @drv: the controller
535
- * @msg: the data to be written to the controller
748
+ * Checks if any of the AMCs are busy in handling ACTIVE sets.
749
+ * This is called from the last cpu powering down before flushing
750
+ * SLEEP and WAKE sets. If AMCs are busy, controller can not enter
751
+ * power collapse, so deny from the last cpu's pm notification.
536752  *
537
- * There is no response returned for writing the request to the controller.
753
+ * Context: Must be called with the drv->lock held.
754
+ *
755
+ * Return:
756
+ * * False		- AMCs are idle
757
+ * * True		- AMCs are busy
538758  */
539
-int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)
759
+static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)
540760 {
541
-	if (!msg || !msg->cmds || !msg->num_cmds ||
542
-	    msg->num_cmds > MAX_RPMH_PAYLOAD) {
543
-		pr_err("Payload error\n");
544
-		return -EINVAL;
761
+	int m;
762
+	struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];
763
+
764
+	/*
765
+	 * If we made an active request on a RSC that does not have a
766
+	 * dedicated TCS for active state use, then re-purposed wake TCSes
767
+	 * should be checked for not busy, because we used wake TCSes for
768
+	 * active requests in this case.
769
+	 */
770
+	if (!tcs->num_tcs)
771
+		tcs = &drv->tcs[WAKE_TCS];
772
+
773
+	for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) {
774
+		if (!tcs_is_free(drv, m))
775
+			return true;
545776 	}
546777 
547
-	/* Data sent to this API will not be sent immediately */
548
-	if (msg->state == RPMH_ACTIVE_ONLY_STATE)
549
-		return -EINVAL;
778
+	return false;
779
+}
550780 
551
-	return tcs_ctrl_write(drv, msg);
781
+/**
782
+ * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
783
+ * @nfb:    Pointer to the notifier block in struct rsc_drv.
784
+ * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT.
785
+ * @v:      Unused
786
+ *
787
+ * This function is given to cpu_pm_register_notifier so we can be informed
788
+ * about when CPUs go down. When all CPUs go down we know no more active
789
+ * transfers will be started so we write sleep/wake sets. This function gets
790
+ * called from cpuidle code paths and also at system suspend time.
791
+ *
792
+ * If its last CPU going down and AMCs are not busy then writes cached sleep
793
+ * and wake messages to TCSes. The firmware then takes care of triggering
794
+ * them when entering deepest low power modes.
795
+ *
796
+ * Return: See cpu_pm_register_notifier()
797
+ */
798
+static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb,
799
+				    unsigned long action, void *v)
800
+{
801
+	struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);
802
+	int ret = NOTIFY_OK;
803
+	int cpus_in_pm;
804
+
805
+	switch (action) {
806
+	case CPU_PM_ENTER:
807
+		cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);
808
+		/*
809
+		 * NOTE: comments for num_online_cpus() point out that it's
810
+		 * only a snapshot so we need to be careful. It should be OK
811
+		 * for us to use, though.  It's important for us not to miss
812
+		 * if we're the last CPU going down so it would only be a
813
+		 * problem if a CPU went offline right after we did the check
814
+		 * AND that CPU was not idle AND that CPU was the last non-idle
815
+		 * CPU. That can't happen. CPUs would have to come out of idle
816
+		 * before the CPU could go offline.
817
+		 */
818
+		if (cpus_in_pm < num_online_cpus())
819
+			return NOTIFY_OK;
820
+		break;
821
+	case CPU_PM_ENTER_FAILED:
822
+	case CPU_PM_EXIT:
823
+		atomic_dec(&drv->cpus_in_pm);
824
+		return NOTIFY_OK;
825
+	default:
826
+		return NOTIFY_DONE;
827
+	}
828
+
829
+	/*
830
+	 * It's likely we're on the last CPU. Grab the drv->lock and write
831
+	 * out the sleep/wake commands to RPMH hardware. Grabbing the lock
832
+	 * means that if we race with another CPU coming up we are still
833
+	 * guaranteed to be safe. If another CPU came up just after we checked
834
+	 * and has grabbed the lock or started an active transfer then we'll
835
+	 * notice we're busy and abort. If another CPU comes up after we start
836
+	 * flushing it will be blocked from starting an active transfer until
837
+	 * we're done flushing. If another CPU starts an active transfer after
838
+	 * we release the lock we're still OK because we're no longer the last
839
+	 * CPU.
840
+	 */
841
+	if (spin_trylock(&drv->lock)) {
842
+		if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))
843
+			ret = NOTIFY_BAD;
844
+		spin_unlock(&drv->lock);
845
+	} else {
846
+		/* Another CPU must be up */
847
+		return NOTIFY_OK;
848
+	}
849
+
850
+	if (ret == NOTIFY_BAD) {
851
+		/* Double-check if we're here because someone else is up */
852
+		if (cpus_in_pm < num_online_cpus())
853
+			ret = NOTIFY_OK;
854
+		else
855
+			/* We won't be called w/ CPU_PM_ENTER_FAILED */
856
+			atomic_dec(&drv->cpus_in_pm);
857
+	}
858
+
859
+	return ret;
552860 }
553861 
554862 static int rpmh_probe_tcs_config(struct platform_device *pdev,
555
-				 struct rsc_drv *drv)
863
+				 struct rsc_drv *drv, void __iomem *base)
556864 {
557865 	struct tcs_type_config {
558866 		u32 type;
....@@ -562,15 +870,6 @@
562870 	u32 config, max_tcs, ncpt, offset;
563871 	int i, ret, n, st = 0;
564872 	struct tcs_group *tcs;
565
-	struct resource *res;
566
-	void __iomem *base;
567
-	char drv_id[10] = {0};
568
-
569
-	snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);
570
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id);
571
-	base = devm_ioremap_resource(&pdev->dev, res);
572
-	if (IS_ERR(base))
573
-		return PTR_ERR(base);
574873 
575874 	ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset);
576875 	if (ret)
....@@ -614,7 +913,6 @@
614913 		tcs->type = tcs_cfg[i].type;
615914 		tcs->num_tcs = tcs_cfg[i].n;
616915 		tcs->ncpt = ncpt;
617
-		spin_lock_init(&tcs->lock);
618916 
619917 		if (!tcs->num_tcs || tcs->type == CONTROL_TCS)
620918 			continue;
....@@ -626,19 +924,6 @@
626924 		tcs->mask = ((1 << tcs->num_tcs) - 1) << st;
627925 		tcs->offset = st;
628926 		st += tcs->num_tcs;
629
-
630
-		/*
631
-		 * Allocate memory to cache sleep and wake requests to
632
-		 * avoid reading TCS register memory.
633
-		 */
634
-		if (tcs->type == ACTIVE_TCS)
635
-			continue;
636
-
637
-		tcs->cmd_cache = devm_kcalloc(&pdev->dev,
638
-					      tcs->num_tcs * ncpt, sizeof(u32),
639
-					      GFP_KERNEL);
640
-		if (!tcs->cmd_cache)
641
-			return -ENOMEM;
642927 	}
643928 
644929 	drv->num_tcs = st;
....@@ -650,7 +935,11 @@
650935 {
651936 	struct device_node *dn = pdev->dev.of_node;
652937 	struct rsc_drv *drv;
938
+	struct resource *res;
939
+	char drv_id[10] = {0};
653940 	int ret, irq;
941
+	u32 solver_config;
942
+	void __iomem *base;
654943 
655944 	/*
656945 	 * Even though RPMh doesn't directly use cmd-db, all of its children
....@@ -676,11 +965,18 @@
676965 	if (!drv->name)
677966 		drv->name = dev_name(&pdev->dev);
678967 
679
-	ret = rpmh_probe_tcs_config(pdev, drv);
968
+	snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);
969
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id);
970
+	base = devm_ioremap_resource(&pdev->dev, res);
971
+	if (IS_ERR(base))
972
+		return PTR_ERR(base);
973
+
974
+	ret = rpmh_probe_tcs_config(pdev, drv, base);
680975 	if (ret)
681976 		return ret;
682977 
683978 	spin_lock_init(&drv->lock);
979
+	init_waitqueue_head(&drv->tcs_wait);
684980 	bitmap_zero(drv->tcs_in_use, MAX_TCS_NR);
685981 
686982 	irq = platform_get_irq(pdev, drv->id);
....@@ -693,8 +989,22 @@
693989 	if (ret)
694990 		return ret;
695991 
992
+	/*
993
+	 * CPU PM notification are not required for controllers that support
994
+	 * 'HW solver' mode where they can be in autonomous mode executing low
995
+	 * power mode to power down.
996
+	 */
997
+	solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG);
998
+	solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT;
999
+	solver_config = solver_config >> DRV_HW_SOLVER_SHIFT;
1000
+	if (!solver_config) {
1001
+		drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
1002
+		cpu_pm_register_notifier(&drv->rsc_pm);
1003
+	}
1004
+
6961005 	/* Enable the active TCS to send requests immediately */
697
-	write_tcs_reg(drv, RSC_DRV_IRQ_ENABLE, 0, drv->tcs[ACTIVE_TCS].mask);
1006
+	writel_relaxed(drv->tcs[ACTIVE_TCS].mask,
1007
+		       drv->tcs_base + RSC_DRV_IRQ_ENABLE);
6981008 
6991009 	spin_lock_init(&drv->client.cache_lock);
7001010 	INIT_LIST_HEAD(&drv->client.cache);
....@@ -709,6 +1019,8 @@
7091019 	{ .compatible = "qcom,rpmh-rsc", },
7101020 	{ }
7111021 };
1022
+MODULE_DEVICE_TABLE(of, rpmh_drv_match);
1023
+
7121024 
7131025 static struct platform_driver rpmh_driver = {
7141026 	.probe = rpmh_rsc_probe,
....@@ -725,5 +1037,5 @@
7251037 }
7261038 arch_initcall(rpmh_driver_init);
7271039 
1040
+MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver");
7281041 MODULE_LICENSE("GPL v2");
729
-MODULE_DESCRIPTION("Qualcomm RPM-Hardened (RPMH) Communication driver");