~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,15 +1,13 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* DMA Engine test module
3	4	*
4	5	* Copyright (C) 2007 Atmel Corporation
5	6	* Copyright (C) 2013 Intel Corporation
6		- *
7		- * This program is free software; you can redistribute it and/or modify
8		- * it under the terms of the GNU General Public License version 2 as
9		- * published by the Free Software Foundation.
10	7	*/
11	8	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12	9
	10	+#include <linux/err.h>
13	11	#include <linux/delay.h>
14	12	#include <linux/dma-mapping.h>
15	13	#include <linux/dmaengine.h>
..	..	@@ -26,11 +24,6 @@
26	24	static unsigned int test_buf_size = 16384;
27	25	module_param(test_buf_size, uint, S_IRUGO \| S_IWUSR);
28	26	MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
29		-
30		-static char test_channel[20];
31		-module_param_string(channel, test_channel, sizeof(test_channel),
32		- S_IRUGO \| S_IWUSR);
33		-MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
34	27
35	28	static char test_device[32];
36	29	module_param_string(device, test_device, sizeof(test_device),
..	..	@@ -68,7 +61,7 @@
68	61	"Number of p+q source buffers (default: 3)");
69	62
70	63	static int timeout = 3000;
71		-module_param(timeout, uint, S_IRUGO \| S_IWUSR);
	64	+module_param(timeout, int, S_IRUGO \| S_IWUSR);
72	65	MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
73	66	"Pass -1 for infinite timeout");
74	67
..	..	@@ -84,6 +77,18 @@
84	77	module_param(verbose, bool, S_IRUGO \| S_IWUSR);
85	78	MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
86	79
	80	+static int alignment = -1;
	81	+module_param(alignment, int, 0644);
	82	+MODULE_PARM_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
	83	+
	84	+static unsigned int transfer_size;
	85	+module_param(transfer_size, uint, 0644);
	86	+MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))");
	87	+
	88	+static bool polled;
	89	+module_param(polled, bool, S_IRUGO \| S_IWUSR);
	90	+MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts");
	91	+
87	92	/**
88	93	* struct dmatest_params - test parameters.
89	94	* @buf_size: size of the memcpy test buffer
..	..	@@ -95,6 +100,11 @@
95	100	* @xor_sources: number of xor source buffers
96	101	* @pq_sources: number of p+q source buffers
97	102	* @timeout: transfer timeout in msec, -1 for infinite timeout
	103	+ * @noverify: disable data verification
	104	+ * @norandom: disable random offset setup
	105	+ * @alignment: custom data address alignment taken as 2^alignment
	106	+ * @transfer_size: custom transfer size in bytes
	107	+ * @polled: use polling for completion instead of interrupts
98	108	*/
99	109	struct dmatest_params {
100	110	unsigned int buf_size;
..	..	@@ -108,12 +118,19 @@
108	118	int timeout;
109	119	bool noverify;
110	120	bool norandom;
	121	+ int alignment;
	122	+ unsigned int transfer_size;
	123	+ bool polled;
111	124	};
112	125
113	126	/**
114	127	* struct dmatest_info - test information.
115	128	* @params: test parameters
	129	+ * @channels: channels under test
	130	+ * @nr_channels: number of channels under test
116	131	* @lock: access protection to the fields of this structure
	132	+ * @did_init: module has been initialized completely
	133	+ * @last_error: test has faced configuration issues
117	134	*/
118	135	static struct dmatest_info {
119	136	/* Test parameters */
..	..	@@ -122,6 +139,7 @@
122	139	/* Internal state */
123	140	struct list_head channels;
124	141	unsigned int nr_channels;
	142	+ int last_error;
125	143	struct mutex lock;
126	144	bool did_init;
127	145	} test_info = {
..	..	@@ -138,6 +156,28 @@
138	156	static bool dmatest_run;
139	157	module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO \| S_IWUSR);
140	158	MODULE_PARM_DESC(run, "Run the test (default: false)");
	159	+
	160	+static int dmatest_chan_set(const char val, const struct kernel_param kp);
	161	+static int dmatest_chan_get(char val, const struct kernel_param kp);
	162	+static const struct kernel_param_ops multi_chan_ops = {
	163	+ .set = dmatest_chan_set,
	164	+ .get = dmatest_chan_get,
	165	+};
	166	+
	167	+static char test_channel[20];
	168	+static struct kparam_string newchan_kps = {
	169	+ .string = test_channel,
	170	+ .maxlen = 20,
	171	+};
	172	+module_param_cb(channel, &multi_chan_ops, &newchan_kps, 0644);
	173	+MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
	174	+
	175	+static int dmatest_test_list_get(char val, const struct kernel_param kp);
	176	+static const struct kernel_param_ops test_list_ops = {
	177	+ .get = dmatest_test_list_get,
	178	+};
	179	+module_param_cb(test_list, &test_list_ops, NULL, 0444);
	180	+MODULE_PARM_DESC(test_list, "Print current test list");
141	181
142	182	/* Maximum amount of mismatched bytes in buffer to print */
143	183	#define MAX_ERROR_COUNT 32
..	..	@@ -160,10 +200,24 @@
160	200	#define PATTERN_COUNT_MASK 0x1f
161	201	#define PATTERN_MEMSET_IDX 0x01
162	202
	203	+/* Fixed point arithmetic ops */
	204	+#define FIXPT_SHIFT 8
	205	+#define FIXPNT_MASK 0xFF
	206	+#define FIXPT_TO_INT(a) ((a) >> FIXPT_SHIFT)
	207	+#define INT_TO_FIXPT(a) ((a) << FIXPT_SHIFT)
	208	+#define FIXPT_GET_FRAC(a) ((((a) & FIXPNT_MASK) * 100) >> FIXPT_SHIFT)
	209	+
163	210	/* poor man's completion - we want to use wait_event_freezable() on it */
164	211	struct dmatest_done {
165	212	bool done;
166	213	wait_queue_head_t *wait;
	214	+};
	215	+
	216	+struct dmatest_data {
	217	+ u8 **raw;
	218	+ u8 **aligned;
	219	+ unsigned int cnt;
	220	+ unsigned int off;
167	221	};
168	222
169	223	struct dmatest_thread {
..	..	@@ -171,14 +225,13 @@
171	225	struct dmatest_info *info;
172	226	struct task_struct *task;
173	227	struct dma_chan *chan;
174		- u8 **srcs;
175		- u8 **usrcs;
176		- u8 **dsts;
177		- u8 **udsts;
	228	+ struct dmatest_data src;
	229	+ struct dmatest_data dst;
178	230	enum dma_transaction_type type;
179	231	wait_queue_head_t done_wait;
180	232	struct dmatest_done test_done;
181	233	bool done;
	234	+ bool pending;
182	235	};
183	236
184	237	struct dmatest_chan {
..	..	@@ -198,7 +251,23 @@
198	251	struct dmatest_thread *thread;
199	252
200	253	list_for_each_entry(thread, &dtc->threads, node) {
201		- if (!thread->done)
	254	+ if (!thread->done && !thread->pending)
	255	+ return true;
	256	+ }
	257	+ }
	258	+
	259	+ return false;
	260	+}
	261	+
	262	+static bool is_threaded_test_pending(struct dmatest_info *info)
	263	+{
	264	+ struct dmatest_chan *dtc;
	265	+
	266	+ list_for_each_entry(dtc, &info->channels, node) {
	267	+ struct dmatest_thread *thread;
	268	+
	269	+ list_for_each_entry(thread, &dtc->threads, node) {
	270	+ if (thread->pending)
202	271	return true;
203	272	}
204	273	}
..	..	@@ -386,8 +455,13 @@
386	455	static void result(const char *err, unsigned int n, unsigned int src_off,
387	456	unsigned int dst_off, unsigned int len, unsigned long data)
388	457	{
389		- pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
390		- current->comm, n, err, src_off, dst_off, len, data);
	458	+ if (IS_ERR_VALUE(data)) {
	459	+ pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%ld)\n",
	460	+ current->comm, n, err, src_off, dst_off, len, data);
	461	+ } else {
	462	+ pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
	463	+ current->comm, n, err, src_off, dst_off, len, data);
	464	+ }
391	465	}
392	466
393	467	static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
..	..	@@ -419,13 +493,62 @@
419	493	}
420	494
421	495	per_sec *= val;
	496	+ per_sec = INT_TO_FIXPT(per_sec);
422	497	do_div(per_sec, runtime);
	498	+
423	499	return per_sec;
424	500	}
425	501
426	502	static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
427	503	{
428		- return dmatest_persec(runtime, len >> 10);
	504	+ return FIXPT_TO_INT(dmatest_persec(runtime, len >> 10));
	505	+}
	506	+
	507	+static void __dmatest_free_test_data(struct dmatest_data *d, unsigned int cnt)
	508	+{
	509	+ unsigned int i;
	510	+
	511	+ for (i = 0; i < cnt; i++)
	512	+ kfree(d->raw[i]);
	513	+
	514	+ kfree(d->aligned);
	515	+ kfree(d->raw);
	516	+}
	517	+
	518	+static void dmatest_free_test_data(struct dmatest_data *d)
	519	+{
	520	+ __dmatest_free_test_data(d, d->cnt);
	521	+}
	522	+
	523	+static int dmatest_alloc_test_data(struct dmatest_data *d,
	524	+ unsigned int buf_size, u8 align)
	525	+{
	526	+ unsigned int i = 0;
	527	+
	528	+ d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
	529	+ if (!d->raw)
	530	+ return -ENOMEM;
	531	+
	532	+ d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
	533	+ if (!d->aligned)
	534	+ goto err;
	535	+
	536	+ for (i = 0; i < d->cnt; i++) {
	537	+ d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL);
	538	+ if (!d->raw[i])
	539	+ goto err;
	540	+
	541	+ /* align to alignment restriction */
	542	+ if (align)
	543	+ d->aligned[i] = PTR_ALIGN(d->raw[i], align);
	544	+ else
	545	+ d->aligned[i] = d->raw[i];
	546	+ }
	547	+
	548	+ return 0;
	549	+err:
	550	+ __dmatest_free_test_data(d, i);
	551	+ return -ENOMEM;
429	552	}
430	553
431	554	/*
..	..	@@ -458,14 +581,16 @@
458	581	enum dma_ctrl_flags flags;
459	582	u8 *pq_coefs = NULL;
460	583	int ret;
461		- int src_cnt;
462		- int dst_cnt;
	584	+ unsigned int buf_size;
	585	+ struct dmatest_data *src;
	586	+ struct dmatest_data *dst;
463	587	int i;
464	588	ktime_t ktime, start, diff;
465	589	ktime_t filltime = 0;
466	590	ktime_t comparetime = 0;
467	591	s64 runtime = 0;
468	592	unsigned long long total_len = 0;
	593	+ unsigned long long iops = 0;
469	594	u8 align = 0;
470	595	bool is_memset = false;
471	596	dma_addr_t *srcs;
..	..	@@ -476,95 +601,81 @@
476	601	ret = -ENOMEM;
477	602
478	603	smp_rmb();
	604	+ thread->pending = false;
479	605	info = thread->info;
480	606	params = &info->params;
481	607	chan = thread->chan;
482	608	dev = chan->device;
	609	+ src = &thread->src;
	610	+ dst = &thread->dst;
483	611	if (thread->type == DMA_MEMCPY) {
484		- align = dev->copy_align;
485		- src_cnt = dst_cnt = 1;
	612	+ align = params->alignment < 0 ? dev->copy_align :
	613	+ params->alignment;
	614	+ src->cnt = dst->cnt = 1;
486	615	} else if (thread->type == DMA_MEMSET) {
487		- align = dev->fill_align;
488		- src_cnt = dst_cnt = 1;
	616	+ align = params->alignment < 0 ? dev->fill_align :
	617	+ params->alignment;
	618	+ src->cnt = dst->cnt = 1;
489	619	is_memset = true;
490	620	} else if (thread->type == DMA_XOR) {
491	621	/* force odd to ensure dst = src */
492		- src_cnt = min_odd(params->xor_sources \| 1, dev->max_xor);
493		- dst_cnt = 1;
494		- align = dev->xor_align;
	622	+ src->cnt = min_odd(params->xor_sources \| 1, dev->max_xor);
	623	+ dst->cnt = 1;
	624	+ align = params->alignment < 0 ? dev->xor_align :
	625	+ params->alignment;
495	626	} else if (thread->type == DMA_PQ) {
496	627	/* force odd to ensure dst = src */
497		- src_cnt = min_odd(params->pq_sources \| 1, dma_maxpq(dev, 0));
498		- dst_cnt = 2;
499		- align = dev->pq_align;
	628	+ src->cnt = min_odd(params->pq_sources \| 1, dma_maxpq(dev, 0));
	629	+ dst->cnt = 2;
	630	+ align = params->alignment < 0 ? dev->pq_align :
	631	+ params->alignment;
500	632
501	633	pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
502	634	if (!pq_coefs)
503	635	goto err_thread_type;
504	636
505		- for (i = 0; i < src_cnt; i++)
	637	+ for (i = 0; i < src->cnt; i++)
506	638	pq_coefs[i] = 1;
507	639	} else
508	640	goto err_thread_type;
509	641
510		- thread->srcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL);
511		- if (!thread->srcs)
512		- goto err_srcs;
513		-
514		- thread->usrcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL);
515		- if (!thread->usrcs)
516		- goto err_usrcs;
517		-
518		- for (i = 0; i < src_cnt; i++) {
519		- thread->usrcs[i] = kmalloc(params->buf_size + align,
520		- GFP_KERNEL);
521		- if (!thread->usrcs[i])
522		- goto err_srcbuf;
523		-
524		- /* align srcs to alignment restriction */
525		- if (align)
526		- thread->srcs[i] = PTR_ALIGN(thread->usrcs[i], align);
527		- else
528		- thread->srcs[i] = thread->usrcs[i];
	642	+ /* Check if buffer count fits into map count variable (u8) */
	643	+ if ((src->cnt + dst->cnt) >= 255) {
	644	+ pr_err("too many buffers (%d of 255 supported)\n",
	645	+ src->cnt + dst->cnt);
	646	+ goto err_free_coefs;
529	647	}
530		- thread->srcs[i] = NULL;
531	648
532		- thread->dsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL);
533		- if (!thread->dsts)
534		- goto err_dsts;
535		-
536		- thread->udsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL);
537		- if (!thread->udsts)
538		- goto err_udsts;
539		-
540		- for (i = 0; i < dst_cnt; i++) {
541		- thread->udsts[i] = kmalloc(params->buf_size + align,
542		- GFP_KERNEL);
543		- if (!thread->udsts[i])
544		- goto err_dstbuf;
545		-
546		- /* align dsts to alignment restriction */
547		- if (align)
548		- thread->dsts[i] = PTR_ALIGN(thread->udsts[i], align);
549		- else
550		- thread->dsts[i] = thread->udsts[i];
	649	+ buf_size = params->buf_size;
	650	+ if (1 << align > buf_size) {
	651	+ pr_err("%u-byte buffer too small for %d-byte alignment\n",
	652	+ buf_size, 1 << align);
	653	+ goto err_free_coefs;
551	654	}
552		- thread->dsts[i] = NULL;
	655	+
	656	+ if (dmatest_alloc_test_data(src, buf_size, align) < 0)
	657	+ goto err_free_coefs;
	658	+
	659	+ if (dmatest_alloc_test_data(dst, buf_size, align) < 0)
	660	+ goto err_src;
553	661
554	662	set_user_nice(current, 10);
555	663
556		- srcs = kcalloc(src_cnt, sizeof(dma_addr_t), GFP_KERNEL);
	664	+ srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL);
557	665	if (!srcs)
558		- goto err_dstbuf;
	666	+ goto err_dst;
559	667
560		- dma_pq = kcalloc(dst_cnt, sizeof(dma_addr_t), GFP_KERNEL);
	668	+ dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL);
561	669	if (!dma_pq)
562	670	goto err_srcs_array;
563	671
564	672	/*
565	673	* src and dst buffers are freed by ourselves below
566	674	*/
567		- flags = DMA_CTRL_ACK \| DMA_PREP_INTERRUPT;
	675	+ if (params->polled)
	676	+ flags = DMA_CTRL_ACK;
	677	+ else
	678	+ flags = DMA_CTRL_ACK \| DMA_PREP_INTERRUPT;
568	679
569	680	ktime = ktime_get();
570	681	while (!(kthread_should_stop() \|\|
..	..	@@ -572,86 +683,83 @@
572	683	struct dma_async_tx_descriptor *tx = NULL;
573	684	struct dmaengine_unmap_data *um;
574	685	dma_addr_t *dsts;
575		- unsigned int src_off, dst_off, len;
	686	+ unsigned int len;
576	687
577	688	total_tests++;
578	689
579		- /* Check if buffer count fits into map count variable (u8) */
580		- if ((src_cnt + dst_cnt) >= 255) {
581		- pr_err("too many buffers (%d of 255 supported)\n",
582		- src_cnt + dst_cnt);
583		- break;
	690	+ if (params->transfer_size) {
	691	+ if (params->transfer_size >= buf_size) {
	692	+ pr_err("%u-byte transfer size must be lower than %u-buffer size\n",
	693	+ params->transfer_size, buf_size);
	694	+ break;
	695	+ }
	696	+ len = params->transfer_size;
	697	+ } else if (params->norandom) {
	698	+ len = buf_size;
	699	+ } else {
	700	+ len = dmatest_random() % buf_size + 1;
584	701	}
585	702
586		- if (1 << align > params->buf_size) {
587		- pr_err("%u-byte buffer too small for %d-byte alignment\n",
588		- params->buf_size, 1 << align);
589		- break;
	703	+ /* Do not alter transfer size explicitly defined by user */
	704	+ if (!params->transfer_size) {
	705	+ len = (len >> align) << align;
	706	+ if (!len)
	707	+ len = 1 << align;
590	708	}
591		-
592		- if (params->norandom)
593		- len = params->buf_size;
594		- else
595		- len = dmatest_random() % params->buf_size + 1;
596		-
597		- len = (len >> align) << align;
598		- if (!len)
599		- len = 1 << align;
600		-
601	709	total_len += len;
602	710
603	711	if (params->norandom) {
604		- src_off = 0;
605		- dst_off = 0;
	712	+ src->off = 0;
	713	+ dst->off = 0;
606	714	} else {
607		- src_off = dmatest_random() % (params->buf_size - len + 1);
608		- dst_off = dmatest_random() % (params->buf_size - len + 1);
	715	+ src->off = dmatest_random() % (buf_size - len + 1);
	716	+ dst->off = dmatest_random() % (buf_size - len + 1);
609	717
610		- src_off = (src_off >> align) << align;
611		- dst_off = (dst_off >> align) << align;
	718	+ src->off = (src->off >> align) << align;
	719	+ dst->off = (dst->off >> align) << align;
612	720	}
613	721
614	722	if (!params->noverify) {
615	723	start = ktime_get();
616		- dmatest_init_srcs(thread->srcs, src_off, len,
617		- params->buf_size, is_memset);
618		- dmatest_init_dsts(thread->dsts, dst_off, len,
619		- params->buf_size, is_memset);
	724	+ dmatest_init_srcs(src->aligned, src->off, len,
	725	+ buf_size, is_memset);
	726	+ dmatest_init_dsts(dst->aligned, dst->off, len,
	727	+ buf_size, is_memset);
620	728
621	729	diff = ktime_sub(ktime_get(), start);
622	730	filltime = ktime_add(filltime, diff);
623	731	}
624	732
625		- um = dmaengine_get_unmap_data(dev->dev, src_cnt + dst_cnt,
	733	+ um = dmaengine_get_unmap_data(dev->dev, src->cnt + dst->cnt,
626	734	GFP_KERNEL);
627	735	if (!um) {
628	736	failed_tests++;
629	737	result("unmap data NULL", total_tests,
630		- src_off, dst_off, len, ret);
	738	+ src->off, dst->off, len, ret);
631	739	continue;
632	740	}
633	741
634		- um->len = params->buf_size;
635		- for (i = 0; i < src_cnt; i++) {
636		- void *buf = thread->srcs[i];
	742	+ um->len = buf_size;
	743	+ for (i = 0; i < src->cnt; i++) {
	744	+ void *buf = src->aligned[i];
637	745	struct page *pg = virt_to_page(buf);
638	746	unsigned long pg_off = offset_in_page(buf);
639	747
640	748	um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
641	749	um->len, DMA_TO_DEVICE);
642		- srcs[i] = um->addr[i] + src_off;
	750	+ srcs[i] = um->addr[i] + src->off;
643	751	ret = dma_mapping_error(dev->dev, um->addr[i]);
644	752	if (ret) {
645	753	result("src mapping error", total_tests,
646		- src_off, dst_off, len, ret);
	754	+ src->off, dst->off, len, ret);
647	755	goto error_unmap_continue;
648	756	}
649	757	um->to_cnt++;
650	758	}
651	759	/* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
652		- dsts = &um->addr[src_cnt];
653		- for (i = 0; i < dst_cnt; i++) {
654		- void *buf = thread->dsts[i];
	760	+ dsts = &um->addr[src->cnt];
	761	+ for (i = 0; i < dst->cnt; i++) {
	762	+ void *buf = dst->aligned[i];
655	763	struct page *pg = virt_to_page(buf);
656	764	unsigned long pg_off = offset_in_page(buf);
657	765
..	..	@@ -660,7 +768,7 @@
660	768	ret = dma_mapping_error(dev->dev, dsts[i]);
661	769	if (ret) {
662	770	result("dst mapping error", total_tests,
663		- src_off, dst_off, len, ret);
	771	+ src->off, dst->off, len, ret);
664	772	goto error_unmap_continue;
665	773	}
666	774	um->bidi_cnt++;
..	..	@@ -668,106 +776,119 @@
668	776
669	777	if (thread->type == DMA_MEMCPY)
670	778	tx = dev->device_prep_dma_memcpy(chan,
671		- dsts[0] + dst_off,
	779	+ dsts[0] + dst->off,
672	780	srcs[0], len, flags);
673	781	else if (thread->type == DMA_MEMSET)
674	782	tx = dev->device_prep_dma_memset(chan,
675		- dsts[0] + dst_off,
676		- *(thread->srcs[0] + src_off),
	783	+ dsts[0] + dst->off,
	784	+ *(src->aligned[0] + src->off),
677	785	len, flags);
678	786	else if (thread->type == DMA_XOR)
679	787	tx = dev->device_prep_dma_xor(chan,
680		- dsts[0] + dst_off,
681		- srcs, src_cnt,
	788	+ dsts[0] + dst->off,
	789	+ srcs, src->cnt,
682	790	len, flags);
683	791	else if (thread->type == DMA_PQ) {
684		- for (i = 0; i < dst_cnt; i++)
685		- dma_pq[i] = dsts[i] + dst_off;
	792	+ for (i = 0; i < dst->cnt; i++)
	793	+ dma_pq[i] = dsts[i] + dst->off;
686	794	tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
687		- src_cnt, pq_coefs,
	795	+ src->cnt, pq_coefs,
688	796	len, flags);
689	797	}
690	798
691	799	if (!tx) {
692		- result("prep error", total_tests, src_off,
693		- dst_off, len, ret);
	800	+ result("prep error", total_tests, src->off,
	801	+ dst->off, len, ret);
694	802	msleep(100);
695	803	goto error_unmap_continue;
696	804	}
697	805
698	806	done->done = false;
699		- tx->callback = dmatest_callback;
700		- tx->callback_param = done;
	807	+ if (!params->polled) {
	808	+ tx->callback = dmatest_callback;
	809	+ tx->callback_param = done;
	810	+ }
701	811	cookie = tx->tx_submit(tx);
702	812
703	813	if (dma_submit_error(cookie)) {
704		- result("submit error", total_tests, src_off,
705		- dst_off, len, ret);
	814	+ result("submit error", total_tests, src->off,
	815	+ dst->off, len, ret);
706	816	msleep(100);
707	817	goto error_unmap_continue;
708	818	}
709		- dma_async_issue_pending(chan);
710	819
711		- wait_event_freezable_timeout(thread->done_wait, done->done,
712		- msecs_to_jiffies(params->timeout));
	820	+ if (params->polled) {
	821	+ status = dma_sync_wait(chan, cookie);
	822	+ dmaengine_terminate_sync(chan);
	823	+ if (status == DMA_COMPLETE)
	824	+ done->done = true;
	825	+ } else {
	826	+ dma_async_issue_pending(chan);
713	827
714		- status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
	828	+ wait_event_freezable_timeout(thread->done_wait,
	829	+ done->done,
	830	+ msecs_to_jiffies(params->timeout));
	831	+
	832	+ status = dma_async_is_tx_complete(chan, cookie, NULL,
	833	+ NULL);
	834	+ }
715	835
716	836	if (!done->done) {
717		- dmaengine_unmap_put(um);
718		- result("test timed out", total_tests, src_off, dst_off,
	837	+ result("test timed out", total_tests, src->off, dst->off,
719	838	len, 0);
720	839	goto error_unmap_continue;
721		- } else if (status != DMA_COMPLETE) {
722		- dmaengine_unmap_put(um);
	840	+ } else if (status != DMA_COMPLETE &&
	841	+ !(dma_has_cap(DMA_COMPLETION_NO_ORDER,
	842	+ dev->cap_mask) &&
	843	+ status == DMA_OUT_OF_ORDER)) {
723	844	result(status == DMA_ERROR ?
724	845	"completion error status" :
725		- "completion busy status", total_tests, src_off,
726		- dst_off, len, ret);
	846	+ "completion busy status", total_tests, src->off,
	847	+ dst->off, len, ret);
727	848	goto error_unmap_continue;
728	849	}
729	850
730	851	dmaengine_unmap_put(um);
731	852
732	853	if (params->noverify) {
733		- verbose_result("test passed", total_tests, src_off,
734		- dst_off, len, 0);
	854	+ verbose_result("test passed", total_tests, src->off,
	855	+ dst->off, len, 0);
735	856	continue;
736	857	}
737	858
738	859	start = ktime_get();
739	860	pr_debug("%s: verifying source buffer...\n", current->comm);
740		- error_count = dmatest_verify(thread->srcs, 0, src_off,
	861	+ error_count = dmatest_verify(src->aligned, 0, src->off,
741	862	0, PATTERN_SRC, true, is_memset);
742		- error_count += dmatest_verify(thread->srcs, src_off,
743		- src_off + len, src_off,
	863	+ error_count += dmatest_verify(src->aligned, src->off,
	864	+ src->off + len, src->off,
744	865	PATTERN_SRC \| PATTERN_COPY, true, is_memset);
745		- error_count += dmatest_verify(thread->srcs, src_off + len,
746		- params->buf_size, src_off + len,
	866	+ error_count += dmatest_verify(src->aligned, src->off + len,
	867	+ buf_size, src->off + len,
747	868	PATTERN_SRC, true, is_memset);
748	869
749	870	pr_debug("%s: verifying dest buffer...\n", current->comm);
750		- error_count += dmatest_verify(thread->dsts, 0, dst_off,
	871	+ error_count += dmatest_verify(dst->aligned, 0, dst->off,
751	872	0, PATTERN_DST, false, is_memset);
752	873
753		- error_count += dmatest_verify(thread->dsts, dst_off,
754		- dst_off + len, src_off,
	874	+ error_count += dmatest_verify(dst->aligned, dst->off,
	875	+ dst->off + len, src->off,
755	876	PATTERN_SRC \| PATTERN_COPY, false, is_memset);
756	877
757		- error_count += dmatest_verify(thread->dsts, dst_off + len,
758		- params->buf_size, dst_off + len,
	878	+ error_count += dmatest_verify(dst->aligned, dst->off + len,
	879	+ buf_size, dst->off + len,
759	880	PATTERN_DST, false, is_memset);
760	881
761	882	diff = ktime_sub(ktime_get(), start);
762	883	comparetime = ktime_add(comparetime, diff);
763	884
764	885	if (error_count) {
765		- result("data error", total_tests, src_off, dst_off,
	886	+ result("data error", total_tests, src->off, dst->off,
766	887	len, error_count);
767	888	failed_tests++;
768	889	} else {
769		- verbose_result("test passed", total_tests, src_off,
770		- dst_off, len, 0);
	890	+ verbose_result("test passed", total_tests, src->off,
	891	+ dst->off, len, 0);
771	892	}
772	893
773	894	continue;
..	..	@@ -785,30 +906,22 @@
785	906	kfree(dma_pq);
786	907	err_srcs_array:
787	908	kfree(srcs);
788		-err_dstbuf:
789		- for (i = 0; thread->udsts[i]; i++)
790		- kfree(thread->udsts[i]);
791		- kfree(thread->udsts);
792		-err_udsts:
793		- kfree(thread->dsts);
794		-err_dsts:
795		-err_srcbuf:
796		- for (i = 0; thread->usrcs[i]; i++)
797		- kfree(thread->usrcs[i]);
798		- kfree(thread->usrcs);
799		-err_usrcs:
800		- kfree(thread->srcs);
801		-err_srcs:
	909	+err_dst:
	910	+ dmatest_free_test_data(dst);
	911	+err_src:
	912	+ dmatest_free_test_data(src);
	913	+err_free_coefs:
802	914	kfree(pq_coefs);
803	915	err_thread_type:
804		- pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
	916	+ iops = dmatest_persec(runtime, total_tests);
	917	+ pr_info("%s: summary %u tests, %u failures %llu.%02llu iops %llu KB/s (%d)\n",
805	918	current->comm, total_tests, failed_tests,
806		- dmatest_persec(runtime, total_tests),
	919	+ FIXPT_TO_INT(iops), FIXPT_GET_FRAC(iops),
807	920	dmatest_KBs(runtime, total_len), ret);
808	921
809	922	/* terminate all transfers on specified channels */
810	923	if (ret \|\| failed_tests)
811		- dmaengine_terminate_all(chan);
	924	+ dmaengine_terminate_sync(chan);
812	925
813	926	thread->done = true;
814	927	wake_up(&thread_wait);
..	..	@@ -832,7 +945,7 @@
832	945	}
833	946
834	947	/* terminate all transfers on specified channels */
835		- dmaengine_terminate_all(dtc->chan);
	948	+ dmaengine_terminate_sync(dtc->chan);
836	949
837	950	kfree(dtc);
838	951	}
..	..	@@ -882,7 +995,7 @@
882	995	/* srcbuf and dstbuf are allocated by the thread itself */
883	996	get_task_struct(thread->task);
884	997	list_add_tail(&thread->node, &dtc->threads);
885		- wake_up_process(thread->task);
	998	+ thread->pending = true;
886	999	}
887	1000
888	1001	return i;
..	..	@@ -904,6 +1017,12 @@
904	1017
905	1018	dtc->chan = chan;
906	1019	INIT_LIST_HEAD(&dtc->threads);
	1020	+
	1021	+ if (dma_has_cap(DMA_COMPLETION_NO_ORDER, dma_dev->cap_mask) &&
	1022	+ info->params.polled) {
	1023	+ info->params.polled = false;
	1024	+ pr_warn("DMA_COMPLETION_NO_ORDER, polled disabled\n");
	1025	+ }
907	1026
908	1027	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
909	1028	if (dmatest == 0) {
..	..	@@ -928,7 +1047,7 @@
928	1047	thread_count += cnt > 0 ? cnt : 0;
929	1048	}
930	1049
931		- pr_info("Started %u threads using %s\n",
	1050	+ pr_info("Added %u threads using %s\n",
932	1051	thread_count, dma_chan_name(chan));
933	1052
934	1053	list_add_tail(&dtc->node, &info->channels);
..	..	@@ -939,13 +1058,7 @@
939	1058
940	1059	static bool filter(struct dma_chan chan, void param)
941	1060	{
942		- struct dmatest_params *params = param;
943		-
944		- if (!dmatest_match_channel(params, chan) \|\|
945		- !dmatest_match_device(params, chan->device))
946		- return false;
947		- else
948		- return true;
	1061	+ return dmatest_match_channel(param, chan) && dmatest_match_device(param, chan->device);
949	1062	}
950	1063
951	1064	static void request_channels(struct dmatest_info *info,
..	..	@@ -973,7 +1086,7 @@
973	1086	}
974	1087	}
975	1088
976		-static void run_threaded_test(struct dmatest_info *info)
	1089	+static void add_threaded_test(struct dmatest_info *info)
977	1090	{
978	1091	struct dmatest_params *params = &info->params;
979	1092
..	..	@@ -989,11 +1102,32 @@
989	1102	params->timeout = timeout;
990	1103	params->noverify = noverify;
991	1104	params->norandom = norandom;
	1105	+ params->alignment = alignment;
	1106	+ params->transfer_size = transfer_size;
	1107	+ params->polled = polled;
992	1108
993	1109	request_channels(info, DMA_MEMCPY);
994	1110	request_channels(info, DMA_MEMSET);
995	1111	request_channels(info, DMA_XOR);
996	1112	request_channels(info, DMA_PQ);
	1113	+}
	1114	+
	1115	+static void run_pending_tests(struct dmatest_info *info)
	1116	+{
	1117	+ struct dmatest_chan *dtc;
	1118	+ unsigned int thread_count = 0;
	1119	+
	1120	+ list_for_each_entry(dtc, &info->channels, node) {
	1121	+ struct dmatest_thread *thread;
	1122	+
	1123	+ thread_count = 0;
	1124	+ list_for_each_entry(thread, &dtc->threads, node) {
	1125	+ wake_up_process(thread->task);
	1126	+ thread_count++;
	1127	+ }
	1128	+ pr_info("Started %u threads using %s\n",
	1129	+ thread_count, dma_chan_name(dtc->chan));
	1130	+ }
997	1131	}
998	1132
999	1133	static void stop_threaded_test(struct dmatest_info *info)
..	..	@@ -1012,7 +1146,7 @@
1012	1146	info->nr_channels = 0;
1013	1147	}
1014	1148
1015		-static void restart_threaded_test(struct dmatest_info *info, bool run)
	1149	+static void start_threaded_tests(struct dmatest_info *info)
1016	1150	{
1017	1151	/* we might be called early to set run=, defer running until all
1018	1152	* parameters have been evaluated
..	..	@@ -1020,11 +1154,7 @@
1020	1154	if (!info->did_init)
1021	1155	return;
1022	1156
1023		- /* Stop any running test first */
1024		- stop_threaded_test(info);
1025		-
1026		- /* Run test with new parameters */
1027		- run_threaded_test(info);
	1157	+ run_pending_tests(info);
1028	1158	}
1029	1159
1030	1160	static int dmatest_run_get(char val, const struct kernel_param kp)
..	..	@@ -1035,7 +1165,8 @@
1035	1165	if (is_threaded_test_run(info)) {
1036	1166	dmatest_run = true;
1037	1167	} else {
1038		- stop_threaded_test(info);
	1168	+ if (!is_threaded_test_pending(info))
	1169	+ stop_threaded_test(info);
1039	1170	dmatest_run = false;
1040	1171	}
1041	1172	mutex_unlock(&info->lock);
..	..	@@ -1053,16 +1184,139 @@
1053	1184	if (ret) {
1054	1185	mutex_unlock(&info->lock);
1055	1186	return ret;
	1187	+ } else if (dmatest_run) {
	1188	+ if (!is_threaded_test_pending(info)) {
	1189	+ /*
	1190	+ * We have nothing to run. This can be due to:
	1191	+ */
	1192	+ ret = info->last_error;
	1193	+ if (ret) {
	1194	+ /* 1) Misconfiguration */
	1195	+ pr_err("Channel misconfigured, can't continue\n");
	1196	+ mutex_unlock(&info->lock);
	1197	+ return ret;
	1198	+ } else {
	1199	+ /* 2) We rely on defaults */
	1200	+ pr_info("No channels configured, continue with any\n");
	1201	+ if (!is_threaded_test_run(info))
	1202	+ stop_threaded_test(info);
	1203	+ add_threaded_test(info);
	1204	+ }
	1205	+ }
	1206	+ start_threaded_tests(info);
	1207	+ } else {
	1208	+ stop_threaded_test(info);
1056	1209	}
1057		-
1058		- if (is_threaded_test_run(info))
1059		- ret = -EBUSY;
1060		- else if (dmatest_run)
1061		- restart_threaded_test(info, dmatest_run);
1062	1210
1063	1211	mutex_unlock(&info->lock);
1064	1212
1065	1213	return ret;
	1214	+}
	1215	+
	1216	+static int dmatest_chan_set(const char val, const struct kernel_param kp)
	1217	+{
	1218	+ struct dmatest_info *info = &test_info;
	1219	+ struct dmatest_chan *dtc;
	1220	+ char chan_reset_val[20];
	1221	+ int ret;
	1222	+
	1223	+ mutex_lock(&info->lock);
	1224	+ ret = param_set_copystring(val, kp);
	1225	+ if (ret) {
	1226	+ mutex_unlock(&info->lock);
	1227	+ return ret;
	1228	+ }
	1229	+ /Clear any previously run threads /
	1230	+ if (!is_threaded_test_run(info) && !is_threaded_test_pending(info))
	1231	+ stop_threaded_test(info);
	1232	+ /* Reject channels that are already registered */
	1233	+ if (is_threaded_test_pending(info)) {
	1234	+ list_for_each_entry(dtc, &info->channels, node) {
	1235	+ if (strcmp(dma_chan_name(dtc->chan),
	1236	+ strim(test_channel)) == 0) {
	1237	+ dtc = list_last_entry(&info->channels,
	1238	+ struct dmatest_chan,
	1239	+ node);
	1240	+ strlcpy(chan_reset_val,
	1241	+ dma_chan_name(dtc->chan),
	1242	+ sizeof(chan_reset_val));
	1243	+ ret = -EBUSY;
	1244	+ goto add_chan_err;
	1245	+ }
	1246	+ }
	1247	+ }
	1248	+
	1249	+ add_threaded_test(info);
	1250	+
	1251	+ /* Check if channel was added successfully */
	1252	+ if (!list_empty(&info->channels)) {
	1253	+ /*
	1254	+ * if new channel was not successfully added, revert the
	1255	+ * "test_channel" string to the name of the last successfully
	1256	+ * added channel. exception for when users issues empty string
	1257	+ * to channel parameter.
	1258	+ */
	1259	+ dtc = list_last_entry(&info->channels, struct dmatest_chan, node);
	1260	+ if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0)
	1261	+ && (strcmp("", strim(test_channel)) != 0)) {
	1262	+ ret = -EINVAL;
	1263	+ strlcpy(chan_reset_val, dma_chan_name(dtc->chan),
	1264	+ sizeof(chan_reset_val));
	1265	+ goto add_chan_err;
	1266	+ }
	1267	+
	1268	+ } else {
	1269	+ /* Clear test_channel if no channels were added successfully */
	1270	+ strlcpy(chan_reset_val, "", sizeof(chan_reset_val));
	1271	+ ret = -EBUSY;
	1272	+ goto add_chan_err;
	1273	+ }
	1274	+
	1275	+ info->last_error = ret;
	1276	+ mutex_unlock(&info->lock);
	1277	+
	1278	+ return ret;
	1279	+
	1280	+add_chan_err:
	1281	+ param_set_copystring(chan_reset_val, kp);
	1282	+ info->last_error = ret;
	1283	+ mutex_unlock(&info->lock);
	1284	+
	1285	+ return ret;
	1286	+}
	1287	+
	1288	+static int dmatest_chan_get(char val, const struct kernel_param kp)
	1289	+{
	1290	+ struct dmatest_info *info = &test_info;
	1291	+
	1292	+ mutex_lock(&info->lock);
	1293	+ if (!is_threaded_test_run(info) && !is_threaded_test_pending(info)) {
	1294	+ stop_threaded_test(info);
	1295	+ strlcpy(test_channel, "", sizeof(test_channel));
	1296	+ }
	1297	+ mutex_unlock(&info->lock);
	1298	+
	1299	+ return param_get_string(val, kp);
	1300	+}
	1301	+
	1302	+static int dmatest_test_list_get(char val, const struct kernel_param kp)
	1303	+{
	1304	+ struct dmatest_info *info = &test_info;
	1305	+ struct dmatest_chan *dtc;
	1306	+ unsigned int thread_count = 0;
	1307	+
	1308	+ list_for_each_entry(dtc, &info->channels, node) {
	1309	+ struct dmatest_thread *thread;
	1310	+
	1311	+ thread_count = 0;
	1312	+ list_for_each_entry(thread, &dtc->threads, node) {
	1313	+ thread_count++;
	1314	+ }
	1315	+ pr_info("%u threads using %s\n",
	1316	+ thread_count, dma_chan_name(dtc->chan));
	1317	+ }
	1318	+
	1319	+ return 0;
1066	1320	}
1067	1321
1068	1322	static int __init dmatest_init(void)
..	..	@@ -1072,7 +1326,8 @@
1072	1326
1073	1327	if (dmatest_run) {
1074	1328	mutex_lock(&info->lock);
1075		- run_threaded_test(info);
	1329	+ add_threaded_test(info);
	1330	+ run_pending_tests(info);
1076	1331	mutex_unlock(&info->lock);
1077	1332	}
1078	1333