forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-04 1543e317f1da31b75942316931e8f491a8920811 
 kernel/drivers/s390/cio/vfio_ccw_cp.c
....@@ -8,6 +8,7 @@
88  *            Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>
99  */
1010 
11
+#include <linux/ratelimit.h>
1112 #include <linux/mm.h>
1213 #include <linux/slab.h>
1314 #include <linux/iommu.h>
....@@ -15,12 +16,6 @@
1516 #include <asm/idals.h>
1617 
1718 #include "vfio_ccw_cp.h"
18
-
19
-/*
20
- * Max length for ccw chain.
21
- * XXX: Limit to 256, need to check more?
22
- */
23
-#define CCWCHAIN_LEN_MAX	256
2419 
2520 struct pfn_array {
2621 	/* Starting guest physical I/O address. */
....@@ -33,11 +28,6 @@
3328 	int			pa_nr;
3429 };
3530 
36
-struct pfn_array_table {
37
-	struct pfn_array	*pat_pa;
38
-	int			pat_nr;
39
-};
40
-
4131 struct ccwchain {
4232 	struct list_head	next;
4333 	struct ccw1		*ch_ccw;
....@@ -46,35 +36,29 @@
4636 	/* Count of the valid ccws in chain. */
4737 	int			ch_len;
4838 	/* Pinned PAGEs for the original data. */
49
-	struct pfn_array_table	*ch_pat;
39
+	struct pfn_array	*ch_pa;
5040 };
5141 
5242 /*
53
- * pfn_array_alloc_pin() - alloc memory for PFNs, then pin user pages in memory
43
+ * pfn_array_alloc() - alloc memory for PFNs
5444  * @pa: pfn_array on which to perform the operation
55
- * @mdev: the mediated device to perform pin/unpin operations
5645  * @iova: target guest physical address
5746  * @len: number of bytes that should be pinned from @iova
5847  *
59
- * Attempt to allocate memory for PFNs, and pin user pages in memory.
48
+ * Attempt to allocate memory for PFNs.
6049  *
6150  * Usage of pfn_array:
6251  * We expect (pa_nr == 0) and (pa_iova_pfn == NULL), any field in
6352  * this structure will be filled in by this function.
6453  *
6554  * Returns:
66
- *   Number of pages pinned on success.
67
- *   If @pa->pa_nr is not 0, or @pa->pa_iova_pfn is not NULL initially,
68
- *   returns -EINVAL.
69
- *   If no pages were pinned, returns -errno.
55
+ *         0 if PFNs are allocated
56
+ *   -EINVAL if pa->pa_nr is not initially zero, or pa->pa_iova_pfn is not NULL
57
+ *   -ENOMEM if alloc failed
7058  */
71
-static int pfn_array_alloc_pin(struct pfn_array *pa, struct device *mdev,
72
-			       u64 iova, unsigned int len)
59
+static int pfn_array_alloc(struct pfn_array *pa, u64 iova, unsigned int len)
7360 {
74
-	int i, ret = 0;
75
-
76
-	if (!len)
77
-		return 0;
61
+	int i;
7862 
7963 	if (pa->pa_nr || pa->pa_iova_pfn)
8064 		return -EINVAL;
....@@ -96,8 +80,27 @@
9680 	pa->pa_pfn = pa->pa_iova_pfn + pa->pa_nr;
9781 
9882 	pa->pa_iova_pfn[0] = pa->pa_iova >> PAGE_SHIFT;
99
-	for (i = 1; i < pa->pa_nr; i++)
83
+	pa->pa_pfn[0] = -1ULL;
84
+	for (i = 1; i < pa->pa_nr; i++) {
10085 		pa->pa_iova_pfn[i] = pa->pa_iova_pfn[i - 1] + 1;
86
+		pa->pa_pfn[i] = -1ULL;
87
+	}
88
+
89
+	return 0;
90
+}
91
+
92
+/*
93
+ * pfn_array_pin() - Pin user pages in memory
94
+ * @pa: pfn_array on which to perform the operation
95
+ * @mdev: the mediated device to perform pin operations
96
+ *
97
+ * Returns number of pages pinned upon success.
98
+ * If the pin request partially succeeds, or fails completely,
99
+ * all pages are left unpinned and a negative error value is returned.
100
+ */
101
+static int pfn_array_pin(struct pfn_array *pa, struct device *mdev)
102
+{
103
+	int ret = 0;
101104 
102105 	ret = vfio_pin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr,
103106 			     IOMMU_READ | IOMMU_WRITE, pa->pa_pfn);
....@@ -114,8 +117,6 @@
114117 
115118 err_out:
116119 	pa->pa_nr = 0;
117
-	kfree(pa->pa_iova_pfn);
118
-	pa->pa_iova_pfn = NULL;
119120 
120121 	return ret;
121122 }
....@@ -123,60 +124,30 @@
123124 /* Unpin the pages before releasing the memory. */
124125 static void pfn_array_unpin_free(struct pfn_array *pa, struct device *mdev)
125126 {
126
-	vfio_unpin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr);
127
+	/* Only unpin if any pages were pinned to begin with */
128
+	if (pa->pa_nr)
129
+		vfio_unpin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr);
127130 	pa->pa_nr = 0;
128131 	kfree(pa->pa_iova_pfn);
129132 }
130133 
131
-static int pfn_array_table_init(struct pfn_array_table *pat, int nr)
134
+static bool pfn_array_iova_pinned(struct pfn_array *pa, unsigned long iova)
132135 {
133
-	pat->pat_pa = kcalloc(nr, sizeof(*pat->pat_pa), GFP_KERNEL);
134
-	if (unlikely(ZERO_OR_NULL_PTR(pat->pat_pa))) {
135
-		pat->pat_nr = 0;
136
-		return -ENOMEM;
137
-	}
138
-
139
-	pat->pat_nr = nr;
140
-
141
-	return 0;
142
-}
143
-
144
-static void pfn_array_table_unpin_free(struct pfn_array_table *pat,
145
-				       struct device *mdev)
146
-{
136
+	unsigned long iova_pfn = iova >> PAGE_SHIFT;
147137 	int i;
148138 
149
-	for (i = 0; i < pat->pat_nr; i++)
150
-		pfn_array_unpin_free(pat->pat_pa + i, mdev);
151
-
152
-	if (pat->pat_nr) {
153
-		kfree(pat->pat_pa);
154
-		pat->pat_pa = NULL;
155
-		pat->pat_nr = 0;
156
-	}
157
-}
158
-
159
-static bool pfn_array_table_iova_pinned(struct pfn_array_table *pat,
160
-					unsigned long iova)
161
-{
162
-	struct pfn_array *pa = pat->pat_pa;
163
-	unsigned long iova_pfn = iova >> PAGE_SHIFT;
164
-	int i, j;
165
-
166
-	for (i = 0; i < pat->pat_nr; i++, pa++)
167
-		for (j = 0; j < pa->pa_nr; j++)
168
-			if (pa->pa_iova_pfn[j] == iova_pfn)
169
-				return true;
139
+	for (i = 0; i < pa->pa_nr; i++)
140
+		if (pa->pa_iova_pfn[i] == iova_pfn)
141
+			return true;
170142 
171143 	return false;
172144 }
173
-/* Create the list idal words for a pfn_array_table. */
174
-static inline void pfn_array_table_idal_create_words(
175
-	struct pfn_array_table *pat,
145
+/* Create the list of IDAL words for a pfn_array. */
146
+static inline void pfn_array_idal_create_words(
147
+	struct pfn_array *pa,
176148 	unsigned long *idaws)
177149 {
178
-	struct pfn_array *pa;
179
-	int i, j, k;
150
+	int i;
180151 
181152 	/*
182153 	 * Idal words (execept the first one) rely on the memory being 4k
....@@ -185,18 +156,35 @@
185156 	 * there will be no problem here to simply use the phys to create an
186157 	 * idaw.
187158 	 */
188
-	k = 0;
189
-	for (i = 0; i < pat->pat_nr; i++) {
190
-		pa = pat->pat_pa + i;
191
-		for (j = 0; j < pa->pa_nr; j++) {
192
-			idaws[k] = pa->pa_pfn[j] << PAGE_SHIFT;
193
-			if (k == 0)
194
-				idaws[k] += pa->pa_iova & (PAGE_SIZE - 1);
195
-			k++;
196
-		}
197
-	}
159
+
160
+	for (i = 0; i < pa->pa_nr; i++)
161
+		idaws[i] = pa->pa_pfn[i] << PAGE_SHIFT;
162
+
163
+	/* Adjust the first IDAW, since it may not start on a page boundary */
164
+	idaws[0] += pa->pa_iova & (PAGE_SIZE - 1);
198165 }
199166 
167
+static void convert_ccw0_to_ccw1(struct ccw1 *source, unsigned long len)
168
+{
169
+	struct ccw0 ccw0;
170
+	struct ccw1 *pccw1 = source;
171
+	int i;
172
+
173
+	for (i = 0; i < len; i++) {
174
+		ccw0 = *(struct ccw0 *)pccw1;
175
+		if ((pccw1->cmd_code & 0x0f) == CCW_CMD_TIC) {
176
+			pccw1->cmd_code = CCW_CMD_TIC;
177
+			pccw1->flags = 0;
178
+			pccw1->count = 0;
179
+		} else {
180
+			pccw1->cmd_code = ccw0.cmd_code;
181
+			pccw1->flags = ccw0.flags;
182
+			pccw1->count = ccw0.count;
183
+		}
184
+		pccw1->cda = ccw0.cda;
185
+		pccw1++;
186
+	}
187
+}
200188 
201189 /*
202190  * Within the domain (@mdev), copy @n bytes from a guest physical
....@@ -211,9 +199,15 @@
211199 	int i, ret;
212200 	unsigned long l, m;
213201 
214
-	ret = pfn_array_alloc_pin(&pa, mdev, iova, n);
215
-	if (ret <= 0)
202
+	ret = pfn_array_alloc(&pa, iova, n);
203
+	if (ret < 0)
216204 		return ret;
205
+
206
+	ret = pfn_array_pin(&pa, mdev);
207
+	if (ret < 0) {
208
+		pfn_array_unpin_free(&pa, mdev);
209
+		return ret;
210
+	}
217211 
218212 	l = n;
219213 	for (i = 0; i < pa.pa_nr; i++) {
....@@ -237,53 +231,85 @@
237231 	return l;
238232 }
239233 
240
-static long copy_ccw_from_iova(struct channel_program *cp,
241
-			       struct ccw1 *to, u64 iova,
242
-			       unsigned long len)
243
-{
244
-	struct ccw0 ccw0;
245
-	struct ccw1 *pccw1;
246
-	int ret;
247
-	int i;
248
-
249
-	ret = copy_from_iova(cp->mdev, to, iova, len * sizeof(struct ccw1));
250
-	if (ret)
251
-		return ret;
252
-
253
-	if (!cp->orb.cmd.fmt) {
254
-		pccw1 = to;
255
-		for (i = 0; i < len; i++) {
256
-			ccw0 = *(struct ccw0 *)pccw1;
257
-			if ((pccw1->cmd_code & 0x0f) == CCW_CMD_TIC) {
258
-				pccw1->cmd_code = CCW_CMD_TIC;
259
-				pccw1->flags = 0;
260
-				pccw1->count = 0;
261
-			} else {
262
-				pccw1->cmd_code = ccw0.cmd_code;
263
-				pccw1->flags = ccw0.flags;
264
-				pccw1->count = ccw0.count;
265
-			}
266
-			pccw1->cda = ccw0.cda;
267
-			pccw1++;
268
-		}
269
-	}
270
-
271
-	return ret;
272
-}
273
-
274234 /*
275235  * Helpers to operate ccwchain.
276236  */
277
-#define ccw_is_test(_ccw) (((_ccw)->cmd_code & 0x0F) == 0)
237
+#define ccw_is_read(_ccw) (((_ccw)->cmd_code & 0x03) == 0x02)
238
+#define ccw_is_read_backward(_ccw) (((_ccw)->cmd_code & 0x0F) == 0x0C)
239
+#define ccw_is_sense(_ccw) (((_ccw)->cmd_code & 0x0F) == CCW_CMD_BASIC_SENSE)
278240 
279241 #define ccw_is_noop(_ccw) ((_ccw)->cmd_code == CCW_CMD_NOOP)
280242 
281243 #define ccw_is_tic(_ccw) ((_ccw)->cmd_code == CCW_CMD_TIC)
282244 
283245 #define ccw_is_idal(_ccw) ((_ccw)->flags & CCW_FLAG_IDA)
284
-
246
+#define ccw_is_skip(_ccw) ((_ccw)->flags & CCW_FLAG_SKIP)
285247 
286248 #define ccw_is_chain(_ccw) ((_ccw)->flags & (CCW_FLAG_CC | CCW_FLAG_DC))
249
+
250
+/*
251
+ * ccw_does_data_transfer()
252
+ *
253
+ * Determine whether a CCW will move any data, such that the guest pages
254
+ * would need to be pinned before performing the I/O.
255
+ *
256
+ * Returns 1 if yes, 0 if no.
257
+ */
258
+static inline int ccw_does_data_transfer(struct ccw1 *ccw)
259
+{
260
+	/* If the count field is zero, then no data will be transferred */
261
+	if (ccw->count == 0)
262
+		return 0;
263
+
264
+	/* If the command is a NOP, then no data will be transferred */
265
+	if (ccw_is_noop(ccw))
266
+		return 0;
267
+
268
+	/* If the skip flag is off, then data will be transferred */
269
+	if (!ccw_is_skip(ccw))
270
+		return 1;
271
+
272
+	/*
273
+	 * If the skip flag is on, it is only meaningful if the command
274
+	 * code is a read, read backward, sense, or sense ID.  In those
275
+	 * cases, no data will be transferred.
276
+	 */
277
+	if (ccw_is_read(ccw) || ccw_is_read_backward(ccw))
278
+		return 0;
279
+
280
+	if (ccw_is_sense(ccw))
281
+		return 0;
282
+
283
+	/* The skip flag is on, but it is ignored for this command code. */
284
+	return 1;
285
+}
286
+
287
+/*
288
+ * is_cpa_within_range()
289
+ *
290
+ * @cpa: channel program address being questioned
291
+ * @head: address of the beginning of a CCW chain
292
+ * @len: number of CCWs within the chain
293
+ *
294
+ * Determine whether the address of a CCW (whether a new chain,
295
+ * or the target of a TIC) falls within a range (including the end points).
296
+ *
297
+ * Returns 1 if yes, 0 if no.
298
+ */
299
+static inline int is_cpa_within_range(u32 cpa, u32 head, int len)
300
+{
301
+	u32 tail = head + (len - 1) * sizeof(struct ccw1);
302
+
303
+	return (head <= cpa && cpa <= tail);
304
+}
305
+
306
+static inline int is_tic_within_range(struct ccw1 *ccw, u32 head, int len)
307
+{
308
+	if (!ccw_is_tic(ccw))
309
+		return 0;
310
+
311
+	return is_cpa_within_range(ccw->cda, head, len);
312
+}
287313 
288314 static struct ccwchain *ccwchain_alloc(struct channel_program *cp, int len)
289315 {
....@@ -294,7 +320,7 @@
294320 	/* Make ccw address aligned to 8. */
295321 	size = ((sizeof(*chain) + 7L) & -8L) +
296322 		sizeof(*chain->ch_ccw) * len +
297
-		sizeof(*chain->ch_pat) * len;
323
+		sizeof(*chain->ch_pa) * len;
298324 	chain = kzalloc(size, GFP_DMA | GFP_KERNEL);
299325 	if (!chain)
300326 		return NULL;
....@@ -303,7 +329,7 @@
303329 	chain->ch_ccw = (struct ccw1 *)data;
304330 
305331 	data = (u8 *)(chain->ch_ccw) + sizeof(*chain->ch_ccw) * len;
306
-	chain->ch_pat = (struct pfn_array_table *)data;
332
+	chain->ch_pa = (struct pfn_array *)data;
307333 
308334 	chain->ch_len = len;
309335 
....@@ -323,28 +349,10 @@
323349 {
324350 	struct ccw1 *ccw = chain->ch_ccw + idx;
325351 
326
-	if (ccw_is_test(ccw) || ccw_is_noop(ccw) || ccw_is_tic(ccw))
327
-		return;
328
-	if (!ccw->count)
352
+	if (ccw_is_tic(ccw))
329353 		return;
330354 
331355 	kfree((void *)(u64)ccw->cda);
332
-}
333
-
334
-/* Unpin the pages then free the memory resources. */
335
-static void cp_unpin_free(struct channel_program *cp)
336
-{
337
-	struct ccwchain *chain, *temp;
338
-	int i;
339
-
340
-	list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) {
341
-		for (i = 0; i < chain->ch_len; i++) {
342
-			pfn_array_table_unpin_free(chain->ch_pat + i,
343
-						   cp->mdev);
344
-			ccwchain_cda_free(chain, i);
345
-		}
346
-		ccwchain_free(chain);
347
-	}
348356 }
349357 
350358 /**
....@@ -362,25 +370,9 @@
362370  */
363371 static int ccwchain_calc_length(u64 iova, struct channel_program *cp)
364372 {
365
-	struct ccw1 *ccw, *p;
366
-	int cnt;
373
+	struct ccw1 *ccw = cp->guest_cp;
374
+	int cnt = 0;
367375 
368
-	/*
369
-	 * Copy current chain from guest to host kernel.
370
-	 * Currently the chain length is limited to CCWCHAIN_LEN_MAX (256).
371
-	 * So copying 2K is enough (safe).
372
-	 */
373
-	p = ccw = kcalloc(CCWCHAIN_LEN_MAX, sizeof(*ccw), GFP_KERNEL);
374
-	if (!ccw)
375
-		return -ENOMEM;
376
-
377
-	cnt = copy_ccw_from_iova(cp, ccw, iova, CCWCHAIN_LEN_MAX);
378
-	if (cnt) {
379
-		kfree(ccw);
380
-		return cnt;
381
-	}
382
-
383
-	cnt = 0;
384376 	do {
385377 		cnt++;
386378 
....@@ -389,12 +381,18 @@
389381 		 * orb specified one of the unsupported formats, we defer
390382 		 * checking for IDAWs in unsupported formats to here.
391383 		 */
392
-		if ((!cp->orb.cmd.c64 || cp->orb.cmd.i2k) && ccw_is_idal(ccw)) {
393
-			kfree(p);
384
+		if ((!cp->orb.cmd.c64 || cp->orb.cmd.i2k) && ccw_is_idal(ccw))
394385 			return -EOPNOTSUPP;
395
-		}
396386 
397
-		if ((!ccw_is_chain(ccw)) && (!ccw_is_tic(ccw)))
387
+		/*
388
+		 * We want to keep counting if the current CCW has the
389
+		 * command-chaining flag enabled, or if it is a TIC CCW
390
+		 * that loops back into the current chain.  The latter
391
+		 * is used for device orientation, where the CCW PRIOR to
392
+		 * the TIC can either jump to the TIC or a CCW immediately
393
+		 * after the TIC, depending on the results of its operation.
394
+		 */
395
+		if (!ccw_is_chain(ccw) && !is_tic_within_range(ccw, iova, cnt))
398396 			break;
399397 
400398 		ccw++;
....@@ -403,20 +401,17 @@
403401 	if (cnt == CCWCHAIN_LEN_MAX + 1)
404402 		cnt = -EINVAL;
405403 
406
-	kfree(p);
407404 	return cnt;
408405 }
409406 
410407 static int tic_target_chain_exists(struct ccw1 *tic, struct channel_program *cp)
411408 {
412409 	struct ccwchain *chain;
413
-	u32 ccw_head, ccw_tail;
410
+	u32 ccw_head;
414411 
415412 	list_for_each_entry(chain, &cp->ccwchain_list, next) {
416413 		ccw_head = chain->ch_iova;
417
-		ccw_tail = ccw_head + (chain->ch_len - 1) * sizeof(struct ccw1);
418
-
419
-		if ((ccw_head <= tic->cda) && (tic->cda <= ccw_tail))
414
+		if (is_cpa_within_range(tic->cda, ccw_head, chain->ch_len))
420415 			return 1;
421416 	}
422417 
....@@ -426,17 +421,23 @@
426421 static int ccwchain_loop_tic(struct ccwchain *chain,
427422 			     struct channel_program *cp);
428423 
429
-static int ccwchain_handle_tic(struct ccw1 *tic, struct channel_program *cp)
424
+static int ccwchain_handle_ccw(u32 cda, struct channel_program *cp)
430425 {
431426 	struct ccwchain *chain;
432427 	int len, ret;
433428 
434
-	/* May transfer to an existing chain. */
435
-	if (tic_target_chain_exists(tic, cp))
436
-		return 0;
429
+	/* Copy 2K (the most we support today) of possible CCWs */
430
+	len = copy_from_iova(cp->mdev, cp->guest_cp, cda,
431
+			     CCWCHAIN_LEN_MAX * sizeof(struct ccw1));
432
+	if (len)
433
+		return len;
437434 
438
-	/* Get chain length. */
439
-	len = ccwchain_calc_length(tic->cda, cp);
435
+	/* Convert any Format-0 CCWs to Format-1 */
436
+	if (!cp->orb.cmd.fmt)
437
+		convert_ccw0_to_ccw1(cp->guest_cp, CCWCHAIN_LEN_MAX);
438
+
439
+	/* Count the CCWs in the current chain */
440
+	len = ccwchain_calc_length(cda, cp);
440441 	if (len < 0)
441442 		return len;
442443 
....@@ -444,17 +445,18 @@
444445 	chain = ccwchain_alloc(cp, len);
445446 	if (!chain)
446447 		return -ENOMEM;
447
-	chain->ch_iova = tic->cda;
448
+	chain->ch_iova = cda;
448449 
449
-	/* Copy the new chain from user. */
450
-	ret = copy_ccw_from_iova(cp, chain->ch_ccw, tic->cda, len);
451
-	if (ret) {
452
-		ccwchain_free(chain);
453
-		return ret;
454
-	}
450
+	/* Copy the actual CCWs into the new chain */
451
+	memcpy(chain->ch_ccw, cp->guest_cp, len * sizeof(struct ccw1));
455452 
456453 	/* Loop for tics on this new chain. */
457
-	return ccwchain_loop_tic(chain, cp);
454
+	ret = ccwchain_loop_tic(chain, cp);
455
+
456
+	if (ret)
457
+		ccwchain_free(chain);
458
+
459
+	return ret;
458460 }
459461 
460462 /* Loop for TICs. */
....@@ -469,7 +471,12 @@
469471 		if (!ccw_is_tic(tic))
470472 			continue;
471473 
472
-		ret = ccwchain_handle_tic(tic, cp);
474
+		/* May transfer to an existing chain. */
475
+		if (tic_target_chain_exists(tic, cp))
476
+			continue;
477
+
478
+		/* Build a ccwchain for the next segment */
479
+		ret = ccwchain_handle_ccw(tic->cda, cp);
473480 		if (ret)
474481 			return ret;
475482 	}
....@@ -483,13 +490,11 @@
483490 {
484491 	struct ccw1 *ccw = chain->ch_ccw + idx;
485492 	struct ccwchain *iter;
486
-	u32 ccw_head, ccw_tail;
493
+	u32 ccw_head;
487494 
488495 	list_for_each_entry(iter, &cp->ccwchain_list, next) {
489496 		ccw_head = iter->ch_iova;
490
-		ccw_tail = ccw_head + (iter->ch_len - 1) * sizeof(struct ccw1);
491
-
492
-		if ((ccw_head <= ccw->cda) && (ccw->cda <= ccw_tail)) {
497
+		if (is_cpa_within_range(ccw->cda, ccw_head, iter->ch_len)) {
493498 			ccw->cda = (__u32) (addr_t) (((char *)iter->ch_ccw) +
494499 						     (ccw->cda - ccw_head));
495500 			return 0;
....@@ -504,115 +509,90 @@
504509 				 struct channel_program *cp)
505510 {
506511 	struct ccw1 *ccw;
507
-	struct pfn_array_table *pat;
512
+	struct pfn_array *pa;
513
+	u64 iova;
508514 	unsigned long *idaws;
509515 	int ret;
516
+	int bytes = 1;
517
+	int idaw_nr, idal_len;
518
+	int i;
510519 
511520 	ccw = chain->ch_ccw + idx;
512521 
513
-	if (!ccw->count) {
514
-		/*
515
-		 * We just want the translation result of any direct ccw
516
-		 * to be an IDA ccw, so let's add the IDA flag for it.
517
-		 * Although the flag will be ignored by firmware.
518
-		 */
519
-		ccw->flags |= CCW_FLAG_IDA;
520
-		return 0;
522
+	if (ccw->count)
523
+		bytes = ccw->count;
524
+
525
+	/* Calculate size of IDAL */
526
+	if (ccw_is_idal(ccw)) {
527
+		/* Read first IDAW to see if it's 4K-aligned or not. */
528
+		/* All subsequent IDAws will be 4K-aligned. */
529
+		ret = copy_from_iova(cp->mdev, &iova, ccw->cda, sizeof(iova));
530
+		if (ret)
531
+			return ret;
532
+	} else {
533
+		iova = ccw->cda;
534
+	}
535
+	idaw_nr = idal_nr_words((void *)iova, bytes);
536
+	idal_len = idaw_nr * sizeof(*idaws);
537
+
538
+	/* Allocate an IDAL from host storage */
539
+	idaws = kcalloc(idaw_nr, sizeof(*idaws), GFP_DMA | GFP_KERNEL);
540
+	if (!idaws) {
541
+		ret = -ENOMEM;
542
+		goto out_init;
521543 	}
522544 
523545 	/*
524
-	 * Pin data page(s) in memory.
525
-	 * The number of pages actually is the count of the idaws which will be
526
-	 * needed when translating a direct ccw to a idal ccw.
546
+	 * Allocate an array of pfn's for pages to pin/translate.
547
+	 * The number of pages is actually the count of the idaws
548
+	 * required for the data transfer, since we only only support
549
+	 * 4K IDAWs today.
527550 	 */
528
-	pat = chain->ch_pat + idx;
529
-	ret = pfn_array_table_init(pat, 1);
530
-	if (ret)
531
-		goto out_init;
532
-
533
-	ret = pfn_array_alloc_pin(pat->pat_pa, cp->mdev, ccw->cda, ccw->count);
551
+	pa = chain->ch_pa + idx;
552
+	ret = pfn_array_alloc(pa, iova, bytes);
534553 	if (ret < 0)
535
-		goto out_unpin;
554
+		goto out_free_idaws;
536555 
537
-	/* Translate this direct ccw to a idal ccw. */
538
-	idaws = kcalloc(ret, sizeof(*idaws), GFP_DMA | GFP_KERNEL);
539
-	if (!idaws) {
540
-		ret = -ENOMEM;
541
-		goto out_unpin;
556
+	if (ccw_is_idal(ccw)) {
557
+		/* Copy guest IDAL into host IDAL */
558
+		ret = copy_from_iova(cp->mdev, idaws, ccw->cda, idal_len);
559
+		if (ret)
560
+			goto out_unpin;
561
+
562
+		/*
563
+		 * Copy guest IDAWs into pfn_array, in case the memory they
564
+		 * occupy is not contiguous.
565
+		 */
566
+		for (i = 0; i < idaw_nr; i++)
567
+			pa->pa_iova_pfn[i] = idaws[i] >> PAGE_SHIFT;
568
+	} else {
569
+		/*
570
+		 * No action is required here; the iova addresses in pfn_array
571
+		 * were initialized sequentially in pfn_array_alloc() beginning
572
+		 * with the contents of ccw->cda.
573
+		 */
542574 	}
575
+
576
+	if (ccw_does_data_transfer(ccw)) {
577
+		ret = pfn_array_pin(pa, cp->mdev);
578
+		if (ret < 0)
579
+			goto out_unpin;
580
+	} else {
581
+		pa->pa_nr = 0;
582
+	}
583
+
543584 	ccw->cda = (__u32) virt_to_phys(idaws);
544585 	ccw->flags |= CCW_FLAG_IDA;
545586 
546
-	pfn_array_table_idal_create_words(pat, idaws);
587
+	/* Populate the IDAL with pinned/translated addresses from pfn */
588
+	pfn_array_idal_create_words(pa, idaws);
547589 
548590 	return 0;
549591 
550592 out_unpin:
551
-	pfn_array_table_unpin_free(pat, cp->mdev);
552
-out_init:
553
-	ccw->cda = 0;
554
-	return ret;
555
-}
556
-
557
-static int ccwchain_fetch_idal(struct ccwchain *chain,
558
-			       int idx,
559
-			       struct channel_program *cp)
560
-{
561
-	struct ccw1 *ccw;
562
-	struct pfn_array_table *pat;
563
-	unsigned long *idaws;
564
-	u64 idaw_iova;
565
-	unsigned int idaw_nr, idaw_len;
566
-	int i, ret;
567
-
568
-	ccw = chain->ch_ccw + idx;
569
-
570
-	if (!ccw->count)
571
-		return 0;
572
-
573
-	/* Calculate size of idaws. */
574
-	ret = copy_from_iova(cp->mdev, &idaw_iova, ccw->cda, sizeof(idaw_iova));
575
-	if (ret)
576
-		return ret;
577
-	idaw_nr = idal_nr_words((void *)(idaw_iova), ccw->count);
578
-	idaw_len = idaw_nr * sizeof(*idaws);
579
-
580
-	/* Pin data page(s) in memory. */
581
-	pat = chain->ch_pat + idx;
582
-	ret = pfn_array_table_init(pat, idaw_nr);
583
-	if (ret)
584
-		goto out_init;
585
-
586
-	/* Translate idal ccw to use new allocated idaws. */
587
-	idaws = kzalloc(idaw_len, GFP_DMA | GFP_KERNEL);
588
-	if (!idaws) {
589
-		ret = -ENOMEM;
590
-		goto out_unpin;
591
-	}
592
-
593
-	ret = copy_from_iova(cp->mdev, idaws, ccw->cda, idaw_len);
594
-	if (ret)
595
-		goto out_free_idaws;
596
-
597
-	ccw->cda = virt_to_phys(idaws);
598
-
599
-	for (i = 0; i < idaw_nr; i++) {
600
-		idaw_iova = *(idaws + i);
601
-
602
-		ret = pfn_array_alloc_pin(pat->pat_pa + i, cp->mdev,
603
-					  idaw_iova, 1);
604
-		if (ret < 0)
605
-			goto out_free_idaws;
606
-	}
607
-
608
-	pfn_array_table_idal_create_words(pat, idaws);
609
-
610
-	return 0;
611
-
593
+	pfn_array_unpin_free(pa, cp->mdev);
612594 out_free_idaws:
613595 	kfree(idaws);
614
-out_unpin:
615
-	pfn_array_table_unpin_free(pat, cp->mdev);
616596 out_init:
617597 	ccw->cda = 0;
618598 	return ret;
....@@ -630,14 +610,8 @@
630610 {
631611 	struct ccw1 *ccw = chain->ch_ccw + idx;
632612 
633
-	if (ccw_is_test(ccw) || ccw_is_noop(ccw))
634
-		return 0;
635
-
636613 	if (ccw_is_tic(ccw))
637614 		return ccwchain_fetch_tic(chain, idx, cp);
638
-
639
-	if (ccw_is_idal(ccw))
640
-		return ccwchain_fetch_idal(chain, idx, cp);
641615 
642616 	return ccwchain_fetch_direct(chain, idx, cp);
643617 }
....@@ -652,56 +626,47 @@
652626  * the target channel program from @orb->cmd.iova to the new ccwchain(s).
653627  *
654628  * Limitations:
655
- * 1. Supports only prefetch enabled mode.
656
- * 2. Supports idal(c64) ccw chaining.
657
- * 3. Supports 4k idaw.
629
+ * 1. Supports idal(c64) ccw chaining.
630
+ * 2. Supports 4k idaw.
658631  *
659632  * Returns:
660633  *   %0 on success and a negative error value on failure.
661634  */
662635 int cp_init(struct channel_program *cp, struct device *mdev, union orb *orb)
663636 {
664
-	u64 iova = orb->cmd.cpa;
665
-	struct ccwchain *chain;
666
-	int len, ret;
637
+	/* custom ratelimit used to avoid flood during guest IPL */
638
+	static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 1);
639
+	int ret;
640
+
641
+	/* this is an error in the caller */
642
+	if (cp->initialized)
643
+		return -EBUSY;
667644 
668645 	/*
669
-	 * XXX:
670
-	 * Only support prefetch enable mode now.
646
+	 * We only support prefetching the channel program. We assume all channel
647
+	 * programs executed by supported guests likewise support prefetching.
648
+	 * Executing a channel program that does not specify prefetching will
649
+	 * typically not cause an error, but a warning is issued to help identify
650
+	 * the problem if something does break.
671651 	 */
672
-	if (!orb->cmd.pfch)
673
-		return -EOPNOTSUPP;
652
+	if (!orb->cmd.pfch && __ratelimit(&ratelimit_state))
653
+		dev_warn(mdev, "Prefetching channel program even though prefetch not specified in ORB");
674654 
675655 	INIT_LIST_HEAD(&cp->ccwchain_list);
676656 	memcpy(&cp->orb, orb, sizeof(*orb));
677657 	cp->mdev = mdev;
678658 
679
-	/* Get chain length. */
680
-	len = ccwchain_calc_length(iova, cp);
681
-	if (len < 0)
682
-		return len;
659
+	/* Build a ccwchain for the first CCW segment */
660
+	ret = ccwchain_handle_ccw(orb->cmd.cpa, cp);
683661 
684
-	/* Alloc mem for the head chain. */
685
-	chain = ccwchain_alloc(cp, len);
686
-	if (!chain)
687
-		return -ENOMEM;
688
-	chain->ch_iova = iova;
662
+	if (!ret) {
663
+		cp->initialized = true;
689664 
690
-	/* Copy the head chain from guest. */
691
-	ret = copy_ccw_from_iova(cp, chain->ch_ccw, iova, len);
692
-	if (ret) {
693
-		ccwchain_free(chain);
694
-		return ret;
665
+		/* It is safe to force: if it was not set but idals used
666
+		 * ccwchain_calc_length would have returned an error.
667
+		 */
668
+		cp->orb.cmd.c64 = 1;
695669 	}
696
-
697
-	/* Now loop for its TICs. */
698
-	ret = ccwchain_loop_tic(chain, cp);
699
-	if (ret)
700
-		cp_unpin_free(cp);
701
-	/* It is safe to force: if not set but idals used
702
-	 * ccwchain_calc_length returns an error.
703
-	 */
704
-	cp->orb.cmd.c64 = 1;
705670 
706671 	return ret;
707672 }
....@@ -717,7 +682,20 @@
717682  */
718683 void cp_free(struct channel_program *cp)
719684 {
720
-	cp_unpin_free(cp);
685
+	struct ccwchain *chain, *temp;
686
+	int i;
687
+
688
+	if (!cp->initialized)
689
+		return;
690
+
691
+	cp->initialized = false;
692
+	list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) {
693
+		for (i = 0; i < chain->ch_len; i++) {
694
+			pfn_array_unpin_free(chain->ch_pa + i, cp->mdev);
695
+			ccwchain_cda_free(chain, i);
696
+		}
697
+		ccwchain_free(chain);
698
+	}
721699 }
722700 
723701 /**
....@@ -762,6 +740,10 @@
762740 	struct ccwchain *chain;
763741 	int len, idx, ret;
764742 
743
+	/* this is an error in the caller */
744
+	if (!cp->initialized)
745
+		return -EINVAL;
746
+
765747 	list_for_each_entry(chain, &cp->ccwchain_list, next) {
766748 		len = chain->ch_len;
767749 		for (idx = 0; idx < len; idx++) {
....@@ -797,6 +779,10 @@
797779 	struct ccwchain *chain;
798780 	struct ccw1 *cpa;
799781 
782
+	/* this is an error in the caller */
783
+	if (!cp->initialized)
784
+		return NULL;
785
+
800786 	orb = &cp->orb;
801787 
802788 	orb->cmd.intparm = intparm;
....@@ -831,7 +817,10 @@
831817 {
832818 	struct ccwchain *chain;
833819 	u32 cpa = scsw->cmd.cpa;
834
-	u32 ccw_head, ccw_tail;
820
+	u32 ccw_head;
821
+
822
+	if (!cp->initialized)
823
+		return;
835824 
836825 	/*
837826 	 * LATER:
....@@ -841,9 +830,11 @@
841830 	 */
842831 	list_for_each_entry(chain, &cp->ccwchain_list, next) {
843832 		ccw_head = (u32)(u64)chain->ch_ccw;
844
-		ccw_tail = (u32)(u64)(chain->ch_ccw + chain->ch_len - 1);
845
-
846
-		if ((ccw_head <= cpa) && (cpa <= ccw_tail)) {
833
+		/*
834
+		 * On successful execution, cpa points just beyond the end
835
+		 * of the chain.
836
+		 */
837
+		if (is_cpa_within_range(cpa, ccw_head, chain->ch_len + 1)) {
847838 			/*
848839 			 * (cpa - ccw_head) is the offset value of the host
849840 			 * physical ccw to its chain head.
....@@ -871,10 +862,12 @@
871862 	struct ccwchain *chain;
872863 	int i;
873864 
865
+	if (!cp->initialized)
866
+		return false;
867
+
874868 	list_for_each_entry(chain, &cp->ccwchain_list, next) {
875869 		for (i = 0; i < chain->ch_len; i++)
876
-			if (pfn_array_table_iova_pinned(chain->ch_pat + i,
877
-							iova))
870
+			if (pfn_array_iova_pinned(chain->ch_pa + i, iova))
878871 				return true;
879872 	}
880873