~hc/RK356X_SDK_RELEASE.git

..	..	@@ -2,7 +2,7 @@
2	2	/*
3	3	* gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4	4	*
5		- * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
	5	+ * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6	6	*
7	7	* Authors: Felipe Balbi <balbi@ti.com>,
8	8	* Sebastian Andrzej Siewior <bigeasy@linutronix.de>
..	..	@@ -46,18 +46,18 @@
46	46	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47	47
48	48	switch (mode) {
49		- case TEST_J:
50		- case TEST_K:
51		- case TEST_SE0_NAK:
52		- case TEST_PACKET:
53		- case TEST_FORCE_EN:
	49	+ case USB_TEST_J:
	50	+ case USB_TEST_K:
	51	+ case USB_TEST_SE0_NAK:
	52	+ case USB_TEST_PACKET:
	53	+ case USB_TEST_FORCE_ENABLE:
54	54	reg \|= mode << 1;
55	55	break;
56	56	default:
57	57	return -EINVAL;
58	58	}
59	59
60		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	60	+ dwc3_gadget_dctl_write_safe(dwc, reg);
61	61
62	62	return 0;
63	63	}
..	..	@@ -95,7 +95,7 @@
95	95	* Wait until device controller is ready. Only applies to 1.94a and
96	96	* later RTL.
97	97	*/
98		- if (dwc->revision >= DWC3_REVISION_194A) {
	98	+ if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99	99	while (--retries) {
100	100	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101	101	if (reg & DWC3_DSTS_DCNRD)
..	..	@@ -111,6 +111,9 @@
111	111	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112	112	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113	113
	114	+ /* set no action before sending new link state change */
	115	+ dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	116	+
114	117	/* set requested state */
115	118	reg \|= DWC3_DCTL_ULSTCHNGREQ(state);
116	119	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
..	..	@@ -119,7 +122,7 @@
119	122	* The following code is racy when called from dwc3_gadget_wakeup,
120	123	* and is not needed, at least on newer versions
121	124	*/
122		- if (dwc->revision >= DWC3_REVISION_194A)
	125	+ if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
123	126	return 0;
124	127
125	128	/* wait for a change in DSTS */
..	..	@@ -135,182 +138,6 @@
135	138
136	139	return -ETIMEDOUT;
137	140	}
138		-
139		-#ifdef CONFIG_ARCH_ROCKCHIP
140		-/**
141		- * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
142		- * @dwc: pointer to our context structure
143		- *
144		- * This function will a best effort FIFO allocation in order
145		- * to improve FIFO usage and throughput, while still allowing
146		- * us to enable as many endpoints as possible.
147		- *
148		- * Keep in mind that this operation will be highly dependent
149		- * on the configured size for RAM1 - which contains TxFifo -,
150		- * the amount of endpoints enabled on coreConsultant tool, and
151		- * the width of the Master Bus.
152		- *
153		- * In the ideal world, we would always be able to satisfy the
154		- * following equation:
155		- *
156		- * ((512 + 2 * MDWIDTH-Bytes) + (Number of IN Endpoints - 1) * \
157		- * (3 * (1024 + MDWIDTH-Bytes) + MDWIDTH-Bytes)) / MDWIDTH-Bytes
158		- *
159		- * Unfortunately, due to many variables that's not always the case.
160		- */
161		-static int dwc3_gadget_resize_tx_fifos(struct dwc3 *dwc)
162		-{
163		- int last_fifo_depth;
164		- int fifo_size, total_size, total_resize = 0;
165		- int mdwidth;
166		- u8 num, fifo_number, num_in_eps;
167		-
168		- /*
169		- * Only support Tx fifos resize for gadget speed <= high speed
170		- * for the time being and do fifo resize operation only once
171		- * when connect done event occurs, because if resize Tx fifos
172		- * during controller transfer data, it may cause controller
173		- * run into abnormal and unrecoverable state.
174		- */
175		- if (!dwc->needs_fifo_resize \|\| dwc->fifo_resize_status)
176		- return 0;
177		-
178		- num_in_eps = DWC3_NUM_IN_EPS(&dwc->hwparams);
179		- mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
180		- /* MDWIDTH is represented in bits, we need it in bytes */
181		- mdwidth >>= 3;
182		- fifo_number = 0;
183		- /* Get the Tx FIFO 0 size and depth */
184		- fifo_size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
185		- last_fifo_depth = DWC3_GTXFIFOSIZ_TXFSTADDR(fifo_size) >> 16;
186		- /* Get the Tx FIFO (num_in_eps - 1) size and depth */
187		- fifo_size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num_in_eps - 1));
188		- /* Get the Tx FIFOs total size */
189		- total_size = (DWC3_GTXFIFOSIZ_TXFSTADDR(fifo_size) >> 16) +
190		- DWC3_GTXFIFOSIZ_TXFDEF(fifo_size) - last_fifo_depth;
191		-
192		- for (num = 0; num < dwc->num_eps; num++) {
193		- struct dwc3_ep *dep = dwc->eps[num];
194		- int mult = 1, maxpacket = 512;
195		- int tmp;
196		-
197		- /* Skip out endpoints */
198		- if (!dep \|\| !dep->direction)
199		- continue;
200		-
201		- switch (dep->endpoint.transfer_type) {
202		- case USB_ENDPOINT_XFER_CONTROL:
203		- if (!dep->endpoint.caps.type_control) {
204		- dev_dbg(dwc->dev, "%s may not be used\n",
205		- dep->name);
206		- goto out;
207		- }
208		-
209		- mult = 1;
210		- if (dwc->gadget.speed <= USB_SPEED_HIGH)
211		- maxpacket = 64;
212		- else
213		- maxpacket = 512;
214		- break;
215		- case USB_ENDPOINT_XFER_ISOC:
216		- if (!dep->endpoint.caps.type_iso) {
217		- dev_WARN(dwc->dev, "%s not support isoc type\n",
218		- dep->name);
219		- goto out;
220		- }
221		-
222		- /*
223		- * Set enough tx fifos for Isochronous endpoints
224		- * to get better performance and more compliance
225		- * with bus latency.
226		- */
227		- maxpacket = dep->endpoint.maxpacket;
228		- if (dwc->gadget.speed <= USB_SPEED_HIGH)
229		- mult = dep->endpoint.mult;
230		- else
231		- mult = dep->endpoint.mult *
232		- dep->endpoint.maxburst;
233		- mult = mult > 0 ? mult * 2 : 3;
234		- if (mult > 6)
235		- mult = 6;
236		- break;
237		- case USB_ENDPOINT_XFER_BULK:
238		- if (!dep->endpoint.caps.type_bulk) {
239		- dev_WARN(dwc->dev, "%s not support bulk type\n",
240		- dep->name);
241		- goto out;
242		- }
243		-
244		- /*
245		- * Set enough tx fifos for Bulk endpoints to get
246		- * better transmission performance.
247		- */
248		- mult = 3;
249		- if (dwc->gadget.speed <= USB_SPEED_HIGH) {
250		- maxpacket = 512;
251		- } else {
252		- if (dep->endpoint.maxburst > mult) {
253		- mult = dep->endpoint.maxburst;
254		- if (mult > 6)
255		- mult = 6;
256		- }
257		- maxpacket = 1024;
258		- }
259		- break;
260		- case USB_ENDPOINT_XFER_INT:
261		- /* Bulk endpoints handle interrupt transfers. */
262		- if (!dep->endpoint.caps.type_int &&
263		- !dep->endpoint.caps.type_bulk) {
264		- dev_WARN(dwc->dev, "%s not support int type\n",
265		- dep->name);
266		- goto out;
267		- }
268		-
269		- /*
270		- * REVIST: we assume that the maxpacket of interrupt
271		- * endpoint is 64 Bytes for MTP and the other functions.
272		- */
273		- mult = 1;
274		- maxpacket = 64;
275		- break;
276		- default:
277		- /*
278		- * This is only possible with faulty memory
279		- * because we checked it already.
280		- */
281		- dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
282		- dep->endpoint.transfer_type);
283		- goto out;
284		- }
285		-
286		- tmp = mult * (maxpacket + mdwidth);
287		- tmp += mdwidth;
288		-
289		- fifo_size = DIV_ROUND_UP(tmp, mdwidth);
290		- total_resize += fifo_size;
291		- fifo_size \|= (last_fifo_depth << 16);
292		-
293		- if (total_resize > total_size) {
294		- dev_WARN(dwc->dev, "Tx FIFO resize overflow!\n");
295		- break;
296		- }
297		-
298		- dev_dbg(dwc->dev, "%s: FIFO Addr %04x Size %d\n",
299		- dep->name, last_fifo_depth, fifo_size & 0xffff);
300		-
301		- dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(fifo_number),
302		- fifo_size);
303		-
304		- last_fifo_depth += (fifo_size & 0xffff);
305		- fifo_number++;
306		- }
307		-
308		-out:
309		- dwc->fifo_resize_status = true;
310		-
311		- return 0;
312		-}
313		-#endif
314	141
315	142	/**
316	143	* dwc3_ep_inc_trb - increment a trb index.
..	..	@@ -400,7 +227,8 @@
400	227	* Caller should take care of locking. Issue @cmd with a given @param to @dwc
401	228	* and wait for its completion.
402	229	*/
403		-int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
	230	+int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
	231	+ u32 param)
404	232	{
405	233	u32 timeout = 500;
406	234	int status = 0;
..	..	@@ -441,7 +269,7 @@
441	269	* Caller should handle locking. This function will issue @cmd with given
442	270	* @params to @dep and wait for its completion.
443	271	*/
444		-int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
	272	+int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
445	273	struct dwc3_gadget_ep_cmd_params *params)
446	274	{
447	275	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
..	..	@@ -463,7 +291,8 @@
463	291	*
464	292	* DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
465	293	*/
466		- if (dwc->gadget.speed <= USB_SPEED_HIGH) {
	294	+ if (dwc->gadget->speed <= USB_SPEED_HIGH \|\|
	295	+ DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) {
467	296	reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
468	297	if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
469	298	saved_config \|= DWC3_GUSB2PHYCFG_SUSPHY;
..	..	@@ -482,19 +311,38 @@
482	311	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
483	312	int link_state;
484	313
	314	+ /*
	315	+ * Initiate remote wakeup if the link state is in U3 when
	316	+ * operating in SS/SSP or L1/L2 when operating in HS/FS. If the
	317	+ * link state is in U1/U2, no remote wakeup is needed. The Start
	318	+ * Transfer command will initiate the link recovery.
	319	+ */
485	320	link_state = dwc3_gadget_get_link_state(dwc);
486		- if (link_state == DWC3_LINK_STATE_U1 \|\|
487		- link_state == DWC3_LINK_STATE_U2 \|\|
488		- link_state == DWC3_LINK_STATE_U3) {
	321	+ switch (link_state) {
	322	+ case DWC3_LINK_STATE_U2:
	323	+ if (dwc->gadget->speed >= USB_SPEED_SUPER)
	324	+ break;
	325	+
	326	+ fallthrough;
	327	+ case DWC3_LINK_STATE_U3:
489	328	ret = __dwc3_gadget_wakeup(dwc);
490	329	dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
491	330	ret);
	331	+ break;
492	332	}
493	333	}
494	334
495		- dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
496		- dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
497		- dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
	335	+ /*
	336	+ * For some commands such as Update Transfer command, DEPCMDPARn
	337	+ * registers are reserved. Since the driver often sends Update Transfer
	338	+ * command, don't write to DEPCMDPARn to avoid register write delays and
	339	+ * improve performance.
	340	+ */
	341	+ if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
	342	+ dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
	343	+ dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
	344	+ dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
	345	+ }
498	346
499	347	/*
500	348	* Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
..	..	@@ -518,6 +366,14 @@
518	366	cmd \|= DWC3_DEPCMD_CMDACT;
519	367
520	368	dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
	369	+
	370	+ if (!(cmd & DWC3_DEPCMD_CMDACT) \|\|
	371	+ (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
	372	+ !(cmd & DWC3_DEPCMD_CMDIOC))) {
	373	+ ret = 0;
	374	+ goto skip_status;
	375	+ }
	376	+
521	377	do {
522	378	reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
523	379	if (!(reg & DWC3_DEPCMD_CMDACT)) {
..	..	@@ -528,6 +384,8 @@
528	384	ret = 0;
529	385	break;
530	386	case DEPEVT_TRANSFER_NO_RESOURCE:
	387	+ dev_WARN(dwc->dev, "No resource for %s\n",
	388	+ dep->name);
531	389	ret = -EINVAL;
532	390	break;
533	391	case DEPEVT_TRANSFER_BUS_EXPIRY:
..	..	@@ -557,11 +415,15 @@
557	415	cmd_status = -ETIMEDOUT;
558	416	}
559	417
	418	+skip_status:
560	419	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
561	420
562		- if (ret == 0 && DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
563		- dep->flags \|= DWC3_EP_TRANSFER_STARTED;
564		- dwc3_gadget_ep_get_transfer_index(dep);
	421	+ if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
	422	+ if (ret == 0)
	423	+ dep->flags \|= DWC3_EP_TRANSFER_STARTED;
	424	+
	425	+ if (ret != -ETIMEDOUT)
	426	+ dwc3_gadget_ep_get_transfer_index(dep);
565	427	}
566	428
567	429	if (saved_config) {
..	..	@@ -572,6 +434,7 @@
572	434
573	435	return ret;
574	436	}
	437	+EXPORT_SYMBOL_GPL(dwc3_send_gadget_ep_cmd);
575	438
576	439	static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
577	440	{
..	..	@@ -587,8 +450,9 @@
587	450	* IN transfers due to a mishandled error condition. Synopsys
588	451	* STAR 9000614252.
589	452	*/
590		- if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
591		- (dwc->gadget.speed >= USB_SPEED_SUPER))
	453	+ if (dep->direction &&
	454	+ !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
	455	+ (dwc->gadget->speed >= USB_SPEED_SUPER))
592	456	cmd \|= DWC3_DEPCMD_CLEARPENDIN;
593	457
594	458	memset(&params, 0, sizeof(params));
..	..	@@ -728,8 +592,9 @@
728	592	\| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
729	593
730	594	/* Burst size is only needed in SuperSpeed mode */
731		- if (dwc->gadget.speed >= USB_SPEED_SUPER) {
	595	+ if (dwc->gadget->speed >= USB_SPEED_SUPER) {
732	596	u32 burst = dep->endpoint.maxburst;
	597	+
733	598	params.param0 \|= DWC3_DEPCFG_BURST_SIZE(burst - 1);
734	599	}
735	600
..	..	@@ -745,6 +610,7 @@
745	610
746	611	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
747	612	params.param1 \|= DWC3_DEPCFG_STREAM_CAPABLE
	613	+ \| DWC3_DEPCFG_XFER_COMPLETE_EN
748	614	\| DWC3_DEPCFG_STREAM_EVENT_EN;
749	615	dep->stream_capable = true;
750	616	}
..	..	@@ -781,7 +647,7 @@
781	647	bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
782	648
783	649	if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
784		- dwc->gadget.speed == USB_SPEED_FULL)
	650	+ dwc->gadget->speed == USB_SPEED_FULL)
785	651	dep->interval = desc->bInterval;
786	652	else
787	653	dep->interval = 1 << (desc->bInterval - 1);
..	..	@@ -790,6 +656,335 @@
790	656	}
791	657
792	658	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
	659	+}
	660	+
	661	+/**
	662	+ * dwc3_gadget_get_tx_fifos_size - Get the txfifos total size
	663	+ * @dwc: pointer to the DWC3 context
	664	+ *
	665	+ * 3-RAM configuration:
	666	+ * RAM0 depth = Descriptor Cache depth
	667	+ * RAM1 depth = TxFIFOs depth
	668	+ * RAM2 depth = RxFIFOs depth
	669	+ *
	670	+ * 2-RAM configuration:
	671	+ * RAM0 depth = Descriptor Cache depth + RxFIFOs depth
	672	+ * RAM1 depth = TxFIFOs depth
	673	+ *
	674	+ * 1-RAM configuration:
	675	+ * RAM0 depth = Descriptor Cache depth + RxFIFOs depth + TxFIFOs depth
	676	+ */
	677	+static int dwc3_gadget_get_tx_fifos_size(struct dwc3 *dwc)
	678	+{
	679	+ int txfifo_depth = 0;
	680	+ int ram0_depth, rxfifo_size;
	681	+
	682	+ /* Get the depth of the TxFIFOs */
	683	+ if (DWC3_NUM_RAMS(dwc->hwparams.hwparams1) > 1) {
	684	+ /* For 2 or 3-RAM, RAM1 contains TxFIFOs */
	685	+ txfifo_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
	686	+ } else {
	687	+ /* For 1-RAM, RAM0 contains Descriptor Cache, RxFIFOs, and TxFIFOs */
	688	+ ram0_depth = DWC3_GHWPARAMS6_RAM0_DEPTH(dwc->hwparams.hwparams6);
	689	+
	690	+ /* All OUT endpoints share a single RxFIFO space */
	691	+ rxfifo_size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
	692	+ if (DWC3_IP_IS(DWC3))
	693	+ txfifo_depth = ram0_depth - DWC3_GRXFIFOSIZ_RXFDEP(rxfifo_size);
	694	+ else
	695	+ txfifo_depth = ram0_depth - DWC31_GRXFIFOSIZ_RXFDEP(rxfifo_size);
	696	+
	697	+ /* The value of GRxFIFOSIZ0[31:16] is the depth of Descriptor Cache */
	698	+ txfifo_depth -= DWC3_GRXFIFOSIZ_RXFSTADDR(rxfifo_size) >> 16;
	699	+ }
	700	+
	701	+ return txfifo_depth;
	702	+}
	703	+
	704	+/**
	705	+ * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
	706	+ * @dwc: pointer to the DWC3 context
	707	+ * @nfifos: number of fifos to calculate for
	708	+ *
	709	+ * Calculates the size value based on the equation below:
	710	+ *
	711	+ * DWC3 revision 280A and prior:
	712	+ * fifo_size = mult * (max_packet / mdwidth) + 1;
	713	+ *
	714	+ * DWC3 revision 290A and onwards:
	715	+ * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
	716	+ *
	717	+ * The max packet size is set to 1024, as the txfifo requirements mainly apply
	718	+ * to super speed USB use cases. However, it is safe to overestimate the fifo
	719	+ * allocations for other scenarios, i.e. high speed USB.
	720	+ */
	721	+static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
	722	+{
	723	+ int max_packet = 1024;
	724	+ int fifo_size;
	725	+ int mdwidth;
	726	+
	727	+ mdwidth = dwc3_mdwidth(dwc);
	728	+
	729	+ /* MDWIDTH is represented in bits, we need it in bytes */
	730	+ mdwidth >>= 3;
	731	+
	732	+ if (DWC3_VER_IS_PRIOR(DWC3, 290A))
	733	+ fifo_size = mult * (max_packet / mdwidth) + 1;
	734	+ else
	735	+ fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
	736	+ return fifo_size;
	737	+}
	738	+
	739	+/**
	740	+ * dwc3_gadget_clear_tx_fifo_size - Clears txfifo allocation
	741	+ * @dwc: pointer to the DWC3 context
	742	+ *
	743	+ * Iterates through all the endpoint registers and clears the previous txfifo
	744	+ * allocations.
	745	+ */
	746	+void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
	747	+{
	748	+ struct dwc3_ep *dep;
	749	+ int fifo_depth;
	750	+ int size;
	751	+ int num;
	752	+
	753	+ if (!dwc->do_fifo_resize)
	754	+ return;
	755	+
	756	+ /* Read ep0IN related TXFIFO size */
	757	+ dep = dwc->eps[1];
	758	+ size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
	759	+ if (DWC3_IP_IS(DWC3))
	760	+ fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
	761	+ else
	762	+ fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
	763	+
	764	+ dwc->last_fifo_depth = fifo_depth;
	765	+ /* Clear existing TXFIFO for all IN eps except ep0 */
	766	+ for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
	767	+ num += 2) {
	768	+ dep = dwc->eps[num];
	769	+ /* Don't change TXFRAMNUM on usb31 version */
	770	+ size = DWC3_IP_IS(DWC3) ? 0 :
	771	+ dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
	772	+ DWC31_GTXFIFOSIZ_TXFRAMNUM;
	773	+
	774	+ dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
	775	+ dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
	776	+ }
	777	+ dwc->num_ep_resized = 0;
	778	+}
	779	+
	780	+/**
	781	+ * __dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for Rockchip platform
	782	+ *
	783	+ * @dep: pointer to dwc3_ep structure
	784	+ *
	785	+ * According to the different USB transfer type and Speed,
	786	+ * this function will a best effort FIFO allocation in order
	787	+ * to improve FIFO usage and throughput, while still allowing
	788	+ * us to enable as many endpoints as possible.
	789	+ */
	790	+static int __dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
	791	+{
	792	+ struct dwc3 *dwc = dep->dwc;
	793	+ u32 fifo_0_start, last_fifo_depth, ram1_depth;
	794	+ u32 fifo_size, maxpacket, mdwidth, mult;
	795	+ u32 tmp;
	796	+
	797	+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
	798	+ /*
	799	+ * Set enough tx fifos for Isochronous endpoints to get better
	800	+ * performance and more compliance with bus latency.
	801	+ */
	802	+ maxpacket = dep->endpoint.maxpacket;
	803	+ if (gadget_is_superspeed(dwc->gadget))
	804	+ mult = dep->endpoint.mult * dep->endpoint.maxburst;
	805	+ else
	806	+ mult = dep->endpoint.mult;
	807	+
	808	+ mult = mult > 0 ? mult * 2 : 3;
	809	+ if (mult > 6)
	810	+ mult = 6;
	811	+ } else if (usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
	812	+ /*
	813	+ * Set enough tx fifos for Bulk endpoints to get
	814	+ * better transmission performance.
	815	+ */
	816	+ mult = 3;
	817	+ if (gadget_is_superspeed(dwc->gadget)) {
	818	+ if (dep->endpoint.maxburst > mult) {
	819	+ mult = dep->endpoint.maxburst;
	820	+ if (mult > 6)
	821	+ mult = 6;
	822	+ }
	823	+ maxpacket = 1024;
	824	+ } else {
	825	+ maxpacket = 512;
	826	+ }
	827	+ } else if (usb_endpoint_xfer_int(dep->endpoint.desc)) {
	828	+ /*
	829	+ * REVIST: we assume that the maxpacket of interrupt
	830	+ * endpoint is 64 Bytes for MTP and the other functions.
	831	+ */
	832	+ mult = 1;
	833	+ maxpacket = 64;
	834	+ } else {
	835	+ goto out;
	836	+ }
	837	+
	838	+ mdwidth = dwc3_mdwidth(dwc);
	839	+ mdwidth >>= 3; /* bits convert to bytes */
	840	+ ram1_depth = dwc3_gadget_get_tx_fifos_size(dwc);
	841	+ last_fifo_depth = dwc->last_fifo_depth;
	842	+
	843	+ /* Calculate the fifo size for this EP */
	844	+ tmp = mult * (maxpacket + mdwidth);
	845	+ tmp += mdwidth;
	846	+ fifo_size = DIV_ROUND_UP(tmp, mdwidth);
	847	+
	848	+ /* Check if TXFIFOs start at non-zero addr */
	849	+ tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
	850	+ fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
	851	+ fifo_size \|= (fifo_0_start + (last_fifo_depth << 16));
	852	+
	853	+ if (DWC3_IP_IS(DWC3))
	854	+ last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
	855	+ else
	856	+ last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
	857	+
	858	+ /* Check fifo size allocation doesn't exceed available RAM size. */
	859	+ if (last_fifo_depth >= ram1_depth) {
	860	+ dev_err(dwc->dev, "Fifosize(0x%x) > RAM size(0x%x) %s depth(0x%x)\n",
	861	+ last_fifo_depth, ram1_depth,
	862	+ dep->endpoint.name, fifo_size & 0xfff);
	863	+ return -ENOMEM;
	864	+ }
	865	+
	866	+ dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
	867	+ dep->flags \|= DWC3_EP_TXFIFO_RESIZED;
	868	+ dwc->last_fifo_depth = last_fifo_depth;
	869	+ dwc->num_ep_resized++;
	870	+
	871	+out:
	872	+ return 0;
	873	+}
	874	+
	875	+/*
	876	+ * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
	877	+ * @dwc: pointer to our context structure
	878	+ *
	879	+ * This function will a best effort FIFO allocation in order
	880	+ * to improve FIFO usage and throughput, while still allowing
	881	+ * us to enable as many endpoints as possible.
	882	+ *
	883	+ * Keep in mind that this operation will be highly dependent
	884	+ * on the configured size for RAM1 - which contains TxFifo -,
	885	+ * the amount of endpoints enabled on coreConsultant tool, and
	886	+ * the width of the Master Bus.
	887	+ *
	888	+ * In general, FIFO depths are represented with the following equation:
	889	+ *
	890	+ * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
	891	+ *
	892	+ * In conjunction with dwc3_gadget_check_config(), this resizing logic will
	893	+ * ensure that all endpoints will have enough internal memory for one max
	894	+ * packet per endpoint.
	895	+ */
	896	+static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
	897	+{
	898	+ struct dwc3 *dwc = dep->dwc;
	899	+ int fifo_0_start;
	900	+ int ram1_depth;
	901	+ int fifo_size;
	902	+ int min_depth;
	903	+ int num_in_ep;
	904	+ int remaining;
	905	+ int num_fifos = 1;
	906	+ int fifo;
	907	+ int tmp;
	908	+
	909	+ if (!dwc->do_fifo_resize)
	910	+ return 0;
	911	+
	912	+ /* resize IN endpoints except ep0 */
	913	+ if (!usb_endpoint_dir_in(dep->endpoint.desc) \|\| dep->number <= 1)
	914	+ return 0;
	915	+
	916	+ /* bail if already resized */
	917	+ if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
	918	+ return 0;
	919	+
	920	+ if (IS_REACHABLE(CONFIG_ARCH_ROCKCHIP))
	921	+ return __dwc3_gadget_resize_tx_fifos(dep);
	922	+
	923	+ ram1_depth = dwc3_gadget_get_tx_fifos_size(dwc);
	924	+
	925	+ if ((dep->endpoint.maxburst > 1 &&
	926	+ usb_endpoint_xfer_bulk(dep->endpoint.desc)) \|\|
	927	+ usb_endpoint_xfer_isoc(dep->endpoint.desc))
	928	+ num_fifos = 3;
	929	+
	930	+ if (dep->endpoint.maxburst > 6 &&
	931	+ (usb_endpoint_xfer_bulk(dep->endpoint.desc) \|\|
	932	+ usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
	933	+ num_fifos = dwc->tx_fifo_resize_max_num;
	934	+
	935	+ /* FIFO size for a single buffer */
	936	+ fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1);
	937	+
	938	+ /* Calculate the number of remaining EPs w/o any FIFO */
	939	+ num_in_ep = dwc->max_cfg_eps;
	940	+ num_in_ep -= dwc->num_ep_resized;
	941	+
	942	+ /* Reserve at least one FIFO for the number of IN EPs */
	943	+ min_depth = num_in_ep * (fifo + 1);
	944	+ remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
	945	+ remaining = max_t(int, 0, remaining);
	946	+ /*
	947	+ * We've already reserved 1 FIFO per EP, so check what we can fit in
	948	+ * addition to it. If there is not enough remaining space, allocate
	949	+ * all the remaining space to the EP.
	950	+ */
	951	+ fifo_size = (num_fifos - 1) * fifo;
	952	+ if (remaining < fifo_size)
	953	+ fifo_size = remaining;
	954	+
	955	+ fifo_size += fifo;
	956	+ /* Last increment according to the TX FIFO size equation */
	957	+ fifo_size++;
	958	+
	959	+ /* Check if TXFIFOs start at non-zero addr */
	960	+ tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
	961	+ fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
	962	+
	963	+ fifo_size \|= (fifo_0_start + (dwc->last_fifo_depth << 16));
	964	+ if (DWC3_IP_IS(DWC3))
	965	+ dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
	966	+ else
	967	+ dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
	968	+
	969	+ /* Check fifo size allocation doesn't exceed available RAM size. */
	970	+ if (dwc->last_fifo_depth >= ram1_depth) {
	971	+ dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
	972	+ dwc->last_fifo_depth, ram1_depth,
	973	+ dep->endpoint.name, fifo_size);
	974	+ if (DWC3_IP_IS(DWC3))
	975	+ fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
	976	+ else
	977	+ fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
	978	+
	979	+ dwc->last_fifo_depth -= fifo_size;
	980	+ return -ENOMEM;
	981	+ }
	982	+
	983	+ dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
	984	+ dep->flags \|= DWC3_EP_TXFIFO_RESIZED;
	985	+ dwc->num_ep_resized++;
	986	+
	987	+ return 0;
793	988	}
794	989
795	990	/**
..	..	@@ -809,6 +1004,10 @@
809	1004	int ret;
810	1005
811	1006	if (!(dep->flags & DWC3_EP_ENABLED)) {
	1007	+ ret = dwc3_gadget_resize_tx_fifos(dep);
	1008	+ if (ret)
	1009	+ return ret;
	1010	+
812	1011	ret = dwc3_gadget_start_config(dep);
813	1012	if (ret)
814	1013	return ret;
..	..	@@ -829,12 +1028,13 @@
829	1028	reg \|= DWC3_DALEPENA_EP(dep->number);
830	1029	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
831	1030
	1031	+ dep->trb_dequeue = 0;
	1032	+ dep->trb_enqueue = 0;
	1033	+
832	1034	if (usb_endpoint_xfer_control(desc))
833	1035	goto out;
834	1036
835	1037	/* Initialize the TRB ring */
836		- dep->trb_dequeue = 0;
837		- dep->trb_enqueue = 0;
838	1038	memset(dep->trb_pool, 0,
839	1039	sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
840	1040
..	..	@@ -852,7 +1052,7 @@
852	1052	* Issue StartTransfer here with no-op TRB so we can always rely on No
853	1053	* Response Update Transfer command.
854	1054	*/
855		- if ((usb_endpoint_xfer_bulk(desc) && !dep->stream_capable) \|\|
	1055	+ if (usb_endpoint_xfer_bulk(desc) \|\|
856	1056	usb_endpoint_xfer_int(desc)) {
857	1057	struct dwc3_gadget_ep_cmd_params params;
858	1058	struct dwc3_trb *trb;
..	..	@@ -871,6 +1071,37 @@
871	1071	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
872	1072	if (ret < 0)
873	1073	return ret;
	1074	+
	1075	+ if (dep->stream_capable) {
	1076	+ /*
	1077	+ * For streams, at start, there maybe a race where the
	1078	+ * host primes the endpoint before the function driver
	1079	+ * queues a request to initiate a stream. In that case,
	1080	+ * the controller will not see the prime to generate the
	1081	+ * ERDY and start stream. To workaround this, issue a
	1082	+ * no-op TRB as normal, but end it immediately. As a
	1083	+ * result, when the function driver queues the request,
	1084	+ * the next START_TRANSFER command will cause the
	1085	+ * controller to generate an ERDY to initiate the
	1086	+ * stream.
	1087	+ */
	1088	+ dwc3_stop_active_transfer(dep, true, true);
	1089	+
	1090	+ /*
	1091	+ * All stream eps will reinitiate stream on NoStream
	1092	+ * rejection until we can determine that the host can
	1093	+ * prime after the first transfer.
	1094	+ *
	1095	+ * However, if the controller is capable of
	1096	+ * TXF_FLUSH_BYPASS, then IN direction endpoints will
	1097	+ * automatically restart the stream without the driver
	1098	+ * initiation.
	1099	+ */
	1100	+ if (!dep->direction \|\|
	1101	+ !(dwc->hwparams.hwparams9 &
	1102	+ DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
	1103	+ dep->flags \|= DWC3_EP_FORCE_RESTART_STREAM;
	1104	+ }
874	1105	}
875	1106
876	1107	out:
..	..	@@ -879,31 +1110,33 @@
879	1110	return 0;
880	1111	}
881	1112
882		-static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
883		- bool interrupt);
884		-static void dwc3_remove_requests(struct dwc3 dwc, struct dwc3_ep dep)
	1113	+void dwc3_remove_requests(struct dwc3 dwc, struct dwc3_ep dep, int status)
885	1114	{
886	1115	struct dwc3_request *req;
887	1116
888	1117	dwc3_stop_active_transfer(dep, true, false);
889	1118
	1119	+ /* If endxfer is delayed, avoid unmapping requests */
	1120	+ if (dep->flags & DWC3_EP_DELAY_STOP)
	1121	+ return;
	1122	+
890	1123	/* - giveback all requests to gadget driver */
891	1124	while (!list_empty(&dep->started_list)) {
892	1125	req = next_request(&dep->started_list);
893	1126
894		- dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
	1127	+ dwc3_gadget_giveback(dep, req, status);
895	1128	}
896	1129
897	1130	while (!list_empty(&dep->pending_list)) {
898	1131	req = next_request(&dep->pending_list);
899	1132
900		- dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
	1133	+ dwc3_gadget_giveback(dep, req, status);
901	1134	}
902	1135
903	1136	while (!list_empty(&dep->cancelled_list)) {
904	1137	req = next_request(&dep->cancelled_list);
905	1138
906		- dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
	1139	+ dwc3_gadget_giveback(dep, req, status);
907	1140	}
908	1141	}
909	1142
..	..	@@ -921,10 +1154,9 @@
921	1154	{
922	1155	struct dwc3 *dwc = dep->dwc;
923	1156	u32 reg;
	1157	+ u32 mask;
924	1158
925	1159	trace_dwc3_gadget_ep_disable(dep);
926		-
927		- dwc3_remove_requests(dwc, dep);
928	1160
929	1161	/* make sure HW endpoint isn't stalled */
930	1162	if (dep->flags & DWC3_EP_STALL)
..	..	@@ -934,9 +1166,19 @@
934	1166	reg &= ~DWC3_DALEPENA_EP(dep->number);
935	1167	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
936	1168
	1169	+ dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
	1170	+
937	1171	dep->stream_capable = false;
938	1172	dep->type = 0;
939		- dep->flags = 0;
	1173	+ mask = DWC3_EP_TXFIFO_RESIZED;
	1174	+ /*
	1175	+ * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is
	1176	+ * set. Do not clear DEP flags, so that the end transfer command will
	1177	+ * be reattempted during the next SETUP stage.
	1178	+ */
	1179	+ if (dep->flags & DWC3_EP_DELAY_STOP)
	1180	+ mask \|= (DWC3_EP_DELAY_STOP \| DWC3_EP_TRANSFER_STARTED);
	1181	+ dep->flags &= mask;
940	1182
941	1183	/* Clear out the ep descriptors for non-ep0 */
942	1184	if (dep->number > 1) {
..	..	@@ -1099,15 +1341,49 @@
1099	1341	return trbs_left;
1100	1342	}
1101	1343
1102		-static void __dwc3_prepare_one_trb(struct dwc3_ep dep, struct dwc3_trb trb,
1103		- dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
1104		- unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
	1344	+/**
	1345	+ * dwc3_prepare_one_trb - setup one TRB from one request
	1346	+ * @dep: endpoint for which this request is prepared
	1347	+ * @req: dwc3_request pointer
	1348	+ * @trb_length: buffer size of the TRB
	1349	+ * @chain: should this TRB be chained to the next?
	1350	+ * @node: only for isochronous endpoints. First TRB needs different type.
	1351	+ * @use_bounce_buffer: set to use bounce buffer
	1352	+ * @must_interrupt: set to interrupt on TRB completion
	1353	+ */
	1354	+static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
	1355	+ struct dwc3_request *req, unsigned int trb_length,
	1356	+ unsigned int chain, unsigned int node, bool use_bounce_buffer,
	1357	+ bool must_interrupt)
1105	1358	{
	1359	+ struct dwc3_trb *trb;
	1360	+ dma_addr_t dma;
	1361	+ unsigned int stream_id = req->request.stream_id;
	1362	+ unsigned int short_not_ok = req->request.short_not_ok;
	1363	+ unsigned int no_interrupt = req->request.no_interrupt;
	1364	+ unsigned int is_last = req->request.is_last;
1106	1365	struct dwc3 *dwc = dep->dwc;
1107		- struct usb_gadget *gadget = &dwc->gadget;
	1366	+ struct usb_gadget *gadget = dwc->gadget;
1108	1367	enum usb_device_speed speed = gadget->speed;
1109	1368
1110		- trb->size = DWC3_TRB_SIZE_LENGTH(length);
	1369	+ if (use_bounce_buffer)
	1370	+ dma = dep->dwc->bounce_addr;
	1371	+ else if (req->request.num_sgs > 0)
	1372	+ dma = sg_dma_address(req->start_sg);
	1373	+ else
	1374	+ dma = req->request.dma;
	1375	+
	1376	+ trb = &dep->trb_pool[dep->trb_enqueue];
	1377	+
	1378	+ if (!req->trb) {
	1379	+ dwc3_gadget_move_started_request(req);
	1380	+ req->trb = trb;
	1381	+ req->trb_dma = dwc3_trb_dma_offset(dep, trb);
	1382	+ }
	1383	+
	1384	+ req->num_trbs++;
	1385	+
	1386	+ trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1111	1387	trb->bpl = lower_32_bits(dma);
1112	1388	trb->bph = upper_32_bits(dma);
1113	1389
..	..	@@ -1147,10 +1423,10 @@
1147	1423	unsigned int mult = 2;
1148	1424	unsigned int maxp = usb_endpoint_maxp(ep->desc);
1149	1425
1150		- if (length <= (2 * maxp))
	1426	+ if (req->request.length <= (2 * maxp))
1151	1427	mult--;
1152	1428
1153		- if (length <= maxp)
	1429	+ if (req->request.length <= maxp)
1154	1430	mult--;
1155	1431
1156	1432	trb->size \|= DWC3_TRB_SIZE_PCM1(mult);
..	..	@@ -1159,8 +1435,8 @@
1159	1435	trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1160	1436	}
1161	1437
1162		- /* always enable Interrupt on Missed ISOC */
1163		- trb->ctrl \|= DWC3_TRB_CTRL_ISP_IMI;
	1438	+ if (!no_interrupt && !chain)
	1439	+ trb->ctrl \|= DWC3_TRB_CTRL_ISP_IMI;
1164	1440	break;
1165	1441
1166	1442	case USB_ENDPOINT_XFER_BULK:
..	..	@@ -1188,12 +1464,13 @@
1188	1464	trb->ctrl \|= DWC3_TRB_CTRL_ISP_IMI;
1189	1465	}
1190	1466
1191		- if ((!no_interrupt && !chain) \|\|
1192		- (dwc3_calc_trbs_left(dep) == 1))
	1467	+ if ((!no_interrupt && !chain) \|\| must_interrupt)
1193	1468	trb->ctrl \|= DWC3_TRB_CTRL_IOC;
1194	1469
1195	1470	if (chain)
1196	1471	trb->ctrl \|= DWC3_TRB_CTRL_CHN;
	1472	+ else if (dep->stream_capable && is_last)
	1473	+ trb->ctrl \|= DWC3_TRB_CTRL_LST;
1197	1474
1198	1475	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1199	1476	trb->ctrl \|= DWC3_TRB_CTRL_SID_SOFN(stream_id);
..	..	@@ -1218,54 +1495,69 @@
1218	1495	trace_dwc3_prepare_trb(dep, trb);
1219	1496	}
1220	1497
1221		-/**
1222		- * dwc3_prepare_one_trb - setup one TRB from one request
1223		- * @dep: endpoint for which this request is prepared
1224		- * @req: dwc3_request pointer
1225		- * @trb_length: buffer size of the TRB
1226		- * @chain: should this TRB be chained to the next?
1227		- * @node: only for isochronous endpoints. First TRB needs different type.
1228		- */
1229		-static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1230		- struct dwc3_request *req, unsigned int trb_length,
1231		- unsigned chain, unsigned node)
	1498	+static bool dwc3_needs_extra_trb(struct dwc3_ep dep, struct dwc3_request req)
1232	1499	{
1233		- struct dwc3_trb *trb;
1234		- dma_addr_t dma;
1235		- unsigned stream_id = req->request.stream_id;
1236		- unsigned short_not_ok = req->request.short_not_ok;
1237		- unsigned no_interrupt = req->request.no_interrupt;
	1500	+ unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
	1501	+ unsigned int rem = req->request.length % maxp;
1238	1502
1239		- if (req->request.num_sgs > 0)
1240		- dma = sg_dma_address(req->start_sg);
1241		- else
1242		- dma = req->request.dma;
	1503	+ if ((req->request.length && req->request.zero && !rem &&
	1504	+ !usb_endpoint_xfer_isoc(dep->endpoint.desc)) \|\|
	1505	+ (!req->direction && rem))
	1506	+ return true;
1243	1507
1244		- trb = &dep->trb_pool[dep->trb_enqueue];
1245		-
1246		- if (!req->trb) {
1247		- dwc3_gadget_move_started_request(req);
1248		- req->trb = trb;
1249		- req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1250		- }
1251		-
1252		- req->num_trbs++;
1253		-
1254		- __dwc3_prepare_one_trb(dep, trb, dma, trb_length, chain, node,
1255		- stream_id, short_not_ok, no_interrupt);
	1508	+ return false;
1256	1509	}
1257	1510
1258		-static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
	1511	+/**
	1512	+ * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
	1513	+ * @dep: The endpoint that the request belongs to
	1514	+ * @req: The request to prepare
	1515	+ * @entry_length: The last SG entry size
	1516	+ * @node: Indicates whether this is not the first entry (for isoc only)
	1517	+ *
	1518	+ * Return the number of TRBs prepared.
	1519	+ */
	1520	+static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
	1521	+ struct dwc3_request *req, unsigned int entry_length,
	1522	+ unsigned int node)
	1523	+{
	1524	+ unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
	1525	+ unsigned int rem = req->request.length % maxp;
	1526	+ unsigned int num_trbs = 1;
	1527	+
	1528	+ if (dwc3_needs_extra_trb(dep, req))
	1529	+ num_trbs++;
	1530	+
	1531	+ if (dwc3_calc_trbs_left(dep) < num_trbs)
	1532	+ return 0;
	1533	+
	1534	+ req->needs_extra_trb = num_trbs > 1;
	1535	+
	1536	+ /* Prepare a normal TRB */
	1537	+ if (req->direction \|\| req->request.length)
	1538	+ dwc3_prepare_one_trb(dep, req, entry_length,
	1539	+ req->needs_extra_trb, node, false, false);
	1540	+
	1541	+ /* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
	1542	+ if ((!req->direction && !req->request.length) \|\| req->needs_extra_trb)
	1543	+ dwc3_prepare_one_trb(dep, req,
	1544	+ req->direction ? 0 : maxp - rem,
	1545	+ false, 1, true, false);
	1546	+
	1547	+ return num_trbs;
	1548	+}
	1549	+
	1550	+static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1259	1551	struct dwc3_request *req)
1260	1552	{
1261	1553	struct scatterlist *sg = req->start_sg;
1262	1554	struct scatterlist *s;
1263	1555	int i;
1264	1556	unsigned int length = req->request.length;
1265		- unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1266		- unsigned int rem = length % maxp;
1267	1557	unsigned int remaining = req->request.num_mapped_sgs
1268	1558	- req->num_queued_sgs;
	1559	+ unsigned int num_trbs = req->num_trbs;
	1560	+ bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1269	1561
1270	1562	/*
1271	1563	* If we resume preparing the request, then get the remaining length of
..	..	@@ -1275,8 +1567,10 @@
1275	1567	length -= sg_dma_len(s);
1276	1568
1277	1569	for_each_sg(sg, s, remaining, i) {
	1570	+ unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1278	1571	unsigned int trb_length;
1279		- unsigned chain = true;
	1572	+ bool must_interrupt = false;
	1573	+ bool last_sg = false;
1280	1574
1281	1575	trb_length = min_t(unsigned int, length, sg_dma_len(s));
1282	1576
..	..	@@ -1290,56 +1584,28 @@
1290	1584	* mapped sg.
1291	1585	*/
1292	1586	if ((i == remaining - 1) \|\| !length)
1293		- chain = false;
	1587	+ last_sg = true;
1294	1588
1295		- if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1296		- struct dwc3 *dwc = dep->dwc;
1297		- struct dwc3_trb *trb;
	1589	+ if (!num_trbs_left)
	1590	+ break;
1298	1591
1299		- req->needs_extra_trb = true;
1300		-
1301		- /* prepare normal TRB */
1302		- dwc3_prepare_one_trb(dep, req, trb_length, true, i);
1303		-
1304		- /* Now prepare one extra TRB to align transfer size */
1305		- trb = &dep->trb_pool[dep->trb_enqueue];
1306		- req->num_trbs++;
1307		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1308		- maxp - rem, false, 1,
1309		- req->request.stream_id,
1310		- req->request.short_not_ok,
1311		- req->request.no_interrupt);
1312		- } else if (req->request.zero && req->request.length &&
1313		- !usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1314		- !rem && !chain) {
1315		- struct dwc3 *dwc = dep->dwc;
1316		- struct dwc3_trb *trb;
1317		-
1318		- req->needs_extra_trb = true;
1319		-
1320		- /* Prepare normal TRB */
1321		- dwc3_prepare_one_trb(dep, req, trb_length, true, i);
1322		-
1323		- /* Prepare one extra TRB to handle ZLP */
1324		- trb = &dep->trb_pool[dep->trb_enqueue];
1325		- req->num_trbs++;
1326		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1327		- !req->direction, 1,
1328		- req->request.stream_id,
1329		- req->request.short_not_ok,
1330		- req->request.no_interrupt);
1331		-
1332		- /* Prepare one more TRB to handle MPS alignment */
1333		- if (!req->direction) {
1334		- trb = &dep->trb_pool[dep->trb_enqueue];
1335		- req->num_trbs++;
1336		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp,
1337		- false, 1, req->request.stream_id,
1338		- req->request.short_not_ok,
1339		- req->request.no_interrupt);
1340		- }
	1592	+ if (last_sg) {
	1593	+ if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
	1594	+ break;
1341	1595	} else {
1342		- dwc3_prepare_one_trb(dep, req, trb_length, chain, i);
	1596	+ /*
	1597	+ * Look ahead to check if we have enough TRBs for the
	1598	+ * next SG entry. If not, set interrupt on this TRB to
	1599	+ * resume preparing the next SG entry when more TRBs are
	1600	+ * free.
	1601	+ */
	1602	+ if (num_trbs_left == 1 \|\| (needs_extra_trb &&
	1603	+ num_trbs_left <= 2 &&
	1604	+ sg_dma_len(sg_next(s)) >= length))
	1605	+ must_interrupt = true;
	1606	+
	1607	+ dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
	1608	+ must_interrupt);
1343	1609	}
1344	1610
1345	1611	/*
..	..	@@ -1349,7 +1615,7 @@
1349	1615	* we have free trbs we can continue queuing from where we
1350	1616	* previously stopped
1351	1617	*/
1352		- if (chain)
	1618	+ if (!last_sg)
1353	1619	req->start_sg = sg_next(s);
1354	1620
1355	1621	req->num_queued_sgs++;
..	..	@@ -1365,65 +1631,17 @@
1365	1631	break;
1366	1632	}
1367	1633
1368		- if (!dwc3_calc_trbs_left(dep))
	1634	+ if (must_interrupt)
1369	1635	break;
1370	1636	}
	1637	+
	1638	+ return req->num_trbs - num_trbs;
1371	1639	}
1372	1640
1373		-static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
	1641	+static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1374	1642	struct dwc3_request *req)
1375	1643	{
1376		- unsigned int length = req->request.length;
1377		- unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1378		- unsigned int rem = length % maxp;
1379		-
1380		- if ((!length \|\| rem) && usb_endpoint_dir_out(dep->endpoint.desc)) {
1381		- struct dwc3 *dwc = dep->dwc;
1382		- struct dwc3_trb *trb;
1383		-
1384		- req->needs_extra_trb = true;
1385		-
1386		- /* prepare normal TRB */
1387		- dwc3_prepare_one_trb(dep, req, length, true, 0);
1388		-
1389		- /* Now prepare one extra TRB to align transfer size */
1390		- trb = &dep->trb_pool[dep->trb_enqueue];
1391		- req->num_trbs++;
1392		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1393		- false, 1, req->request.stream_id,
1394		- req->request.short_not_ok,
1395		- req->request.no_interrupt);
1396		- } else if (req->request.zero && req->request.length &&
1397		- !usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1398		- (IS_ALIGNED(req->request.length, maxp))) {
1399		- struct dwc3 *dwc = dep->dwc;
1400		- struct dwc3_trb *trb;
1401		-
1402		- req->needs_extra_trb = true;
1403		-
1404		- /* prepare normal TRB */
1405		- dwc3_prepare_one_trb(dep, req, length, true, 0);
1406		-
1407		- /* Prepare one extra TRB to handle ZLP */
1408		- trb = &dep->trb_pool[dep->trb_enqueue];
1409		- req->num_trbs++;
1410		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1411		- !req->direction, 1, req->request.stream_id,
1412		- req->request.short_not_ok,
1413		- req->request.no_interrupt);
1414		-
1415		- /* Prepare one more TRB to handle MPS alignment for OUT */
1416		- if (!req->direction) {
1417		- trb = &dep->trb_pool[dep->trb_enqueue];
1418		- req->num_trbs++;
1419		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp,
1420		- false, 1, req->request.stream_id,
1421		- req->request.short_not_ok,
1422		- req->request.no_interrupt);
1423		- }
1424		- } else {
1425		- dwc3_prepare_one_trb(dep, req, length, false, 0);
1426		- }
	1644	+ return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1427	1645	}
1428	1646
1429	1647	/*
..	..	@@ -1433,10 +1651,13 @@
1433	1651	* The function goes through the requests list and sets up TRBs for the
1434	1652	* transfers. The function returns once there are no more TRBs available or
1435	1653	* it runs out of requests.
	1654	+ *
	1655	+ * Returns the number of TRBs prepared or negative errno.
1436	1656	*/
1437		-static void dwc3_prepare_trbs(struct dwc3_ep *dep)
	1657	+static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1438	1658	{
1439	1659	struct dwc3_request req, n;
	1660	+ int ret = 0;
1440	1661
1441	1662	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1442	1663
..	..	@@ -1451,35 +1672,58 @@
1451	1672	* break things.
1452	1673	*/
1453	1674	list_for_each_entry(req, &dep->started_list, list) {
1454		- if (req->num_pending_sgs > 0)
1455		- dwc3_prepare_one_trb_sg(dep, req);
	1675	+ if (req->num_pending_sgs > 0) {
	1676	+ ret = dwc3_prepare_trbs_sg(dep, req);
	1677	+ if (!ret \|\| req->num_pending_sgs)
	1678	+ return ret;
	1679	+ }
1456	1680
1457	1681	if (!dwc3_calc_trbs_left(dep))
1458		- return;
	1682	+ return ret;
	1683	+
	1684	+ /*
	1685	+ * Don't prepare beyond a transfer. In DWC_usb32, its transfer
	1686	+ * burst capability may try to read and use TRBs beyond the
	1687	+ * active transfer instead of stopping.
	1688	+ */
	1689	+ if (dep->stream_capable && req->request.is_last)
	1690	+ return ret;
1459	1691	}
1460	1692
1461	1693	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1462	1694	struct dwc3 *dwc = dep->dwc;
1463		- int ret;
1464	1695
1465	1696	ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1466	1697	dep->direction);
1467	1698	if (ret)
1468		- return;
	1699	+ return ret;
1469	1700
1470	1701	req->sg = req->request.sg;
1471	1702	req->start_sg = req->sg;
1472	1703	req->num_queued_sgs = 0;
1473	1704	req->num_pending_sgs = req->request.num_mapped_sgs;
1474	1705
1475		- if (req->num_pending_sgs > 0)
1476		- dwc3_prepare_one_trb_sg(dep, req);
1477		- else
1478		- dwc3_prepare_one_trb_linear(dep, req);
	1706	+ if (req->num_pending_sgs > 0) {
	1707	+ ret = dwc3_prepare_trbs_sg(dep, req);
	1708	+ if (req->num_pending_sgs)
	1709	+ return ret;
	1710	+ } else {
	1711	+ ret = dwc3_prepare_trbs_linear(dep, req);
	1712	+ }
1479	1713
1480		- if (!dwc3_calc_trbs_left(dep))
1481		- return;
	1714	+ if (!ret \|\| !dwc3_calc_trbs_left(dep))
	1715	+ return ret;
	1716	+
	1717	+ /*
	1718	+ * Don't prepare beyond a transfer. In DWC_usb32, its transfer
	1719	+ * burst capability may try to read and use TRBs beyond the
	1720	+ * active transfer instead of stopping.
	1721	+ */
	1722	+ if (dep->stream_capable && req->request.is_last)
	1723	+ return ret;
1482	1724	}
	1725	+
	1726	+ return ret;
1483	1727	}
1484	1728
1485	1729	static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
..	..	@@ -1492,12 +1736,24 @@
1492	1736	int ret;
1493	1737	u32 cmd;
1494	1738
1495		- if (!dwc3_calc_trbs_left(dep))
1496		- return 0;
	1739	+ /*
	1740	+ * Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
	1741	+ * This happens when we need to stop and restart a transfer such as in
	1742	+ * the case of reinitiating a stream or retrying an isoc transfer.
	1743	+ */
	1744	+ ret = dwc3_prepare_trbs(dep);
	1745	+ if (ret < 0)
	1746	+ return ret;
1497	1747
1498	1748	starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1499	1749
1500		- dwc3_prepare_trbs(dep);
	1750	+ /*
	1751	+ * If there's no new TRB prepared and we don't need to restart a
	1752	+ * transfer, there's no need to update the transfer.
	1753	+ */
	1754	+ if (!ret && !starting)
	1755	+ return ret;
	1756	+
1501	1757	req = next_request(&dep->started_list);
1502	1758	if (!req) {
1503	1759	dep->flags \|= DWC3_EP_PENDING_REQUEST;
..	..	@@ -1525,22 +1781,13 @@
1525	1781	if (ret < 0) {
1526	1782	struct dwc3_request *tmp;
1527	1783
1528		- /*
1529		- * Isochronous endpoints request needs to
1530		- * return directly and retry to transfer next
1531		- * time. Otherwise, it will fail to giveback
1532		- * the req to the udc gadget driver.
1533		- */
1534		- if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1535		- return ret;
1536		-
1537	1784	if (ret == -EAGAIN)
1538	1785	return ret;
1539	1786
1540	1787	dwc3_stop_active_transfer(dep, true, true);
1541	1788
1542	1789	list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1543		- dwc3_gadget_move_cancelled_request(req);
	1790	+ dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1544	1791
1545	1792	/* If ep isn't started, then there's no end transfer pending */
1546	1793	if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
..	..	@@ -1548,6 +1795,9 @@
1548	1795
1549	1796	return ret;
1550	1797	}
	1798	+
	1799	+ if (dep->stream_capable && req->request.is_last)
	1800	+ dep->flags \|= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1551	1801
1552	1802	return 0;
1553	1803	}
..	..	@@ -1558,6 +1808,50 @@
1558	1808
1559	1809	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1560	1810	return DWC3_DSTS_SOFFN(reg);
	1811	+}
	1812	+
	1813	+/**
	1814	+ * __dwc3_stop_active_transfer - stop the current active transfer
	1815	+ * @dep: isoc endpoint
	1816	+ * @force: set forcerm bit in the command
	1817	+ * @interrupt: command complete interrupt after End Transfer command
	1818	+ *
	1819	+ * When setting force, the ForceRM bit will be set. In that case
	1820	+ * the controller won't update the TRB progress on command
	1821	+ * completion. It also won't clear the HWO bit in the TRB.
	1822	+ * The command will also not complete immediately in that case.
	1823	+ */
	1824	+static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
	1825	+{
	1826	+ struct dwc3_gadget_ep_cmd_params params;
	1827	+ u32 cmd;
	1828	+ int ret;
	1829	+
	1830	+ cmd = DWC3_DEPCMD_ENDTRANSFER;
	1831	+ cmd \|= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
	1832	+ cmd \|= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
	1833	+ cmd \|= DWC3_DEPCMD_PARAM(dep->resource_index);
	1834	+ memset(&params, 0, sizeof(params));
	1835	+ ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
	1836	+ /*
	1837	+ * If the End Transfer command was timed out while the device is
	1838	+ * not in SETUP phase, it's possible that an incoming Setup packet
	1839	+ * may prevent the command's completion. Let's retry when the
	1840	+ * ep0state returns to EP0_SETUP_PHASE.
	1841	+ */
	1842	+ if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
	1843	+ dep->flags \|= DWC3_EP_DELAY_STOP;
	1844	+ return 0;
	1845	+ }
	1846	+ WARN_ON_ONCE(ret);
	1847	+ dep->resource_index = 0;
	1848	+
	1849	+ if (!interrupt)
	1850	+ dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
	1851	+ else
	1852	+ dep->flags \|= DWC3_EP_END_TRANSFER_PENDING;
	1853	+
	1854	+ return ret;
1561	1855	}
1562	1856
1563	1857	/**
..	..	@@ -1617,7 +1911,7 @@
1617	1911	* Check if we can start isoc transfer on the next interval or
1618	1912	* 4 uframes in the future with BIT[15:14] as dep->combo_num
1619	1913	*/
1620		- test_frame_number = dep->frame_number & 0x3fff;
	1914	+ test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1621	1915	test_frame_number \|= dep->combo_num << 14;
1622	1916	test_frame_number += max_t(u32, 4, dep->interval);
1623	1917
..	..	@@ -1664,7 +1958,7 @@
1664	1958	else if (test0 && test1)
1665	1959	dep->combo_num = 0;
1666	1960
1667		- dep->frame_number &= 0x3fff;
	1961	+ dep->frame_number &= DWC3_FRNUMBER_MASK;
1668	1962	dep->frame_number \|= dep->combo_num << 14;
1669	1963	dep->frame_number += max_t(u32, 4, dep->interval);
1670	1964
..	..	@@ -1682,26 +1976,24 @@
1682	1976	int ret;
1683	1977	int i;
1684	1978
1685		- if (list_empty(&dep->pending_list)) {
	1979	+ if (list_empty(&dep->pending_list) &&
	1980	+ list_empty(&dep->started_list)) {
1686	1981	dep->flags \|= DWC3_EP_PENDING_REQUEST;
1687	1982	return -EAGAIN;
1688	1983	}
1689	1984
1690		- if (!dwc->dis_start_transfer_quirk && dwc3_is_usb31(dwc) &&
1691		- (dwc->revision <= DWC3_USB31_REVISION_160A \|\|
1692		- (dwc->revision == DWC3_USB31_REVISION_170A &&
1693		- dwc->version_type >= DWC31_VERSIONTYPE_EA01 &&
1694		- dwc->version_type <= DWC31_VERSIONTYPE_EA06))) {
1695		-
1696		- if (dwc->gadget.speed <= USB_SPEED_HIGH && dep->direction)
	1985	+ if (!dwc->dis_start_transfer_quirk &&
	1986	+ (DWC3_VER_IS_PRIOR(DWC31, 170A) \|\|
	1987	+ DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
	1988	+ if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1697	1989	return dwc3_gadget_start_isoc_quirk(dep);
1698	1990	}
1699	1991
1700	1992	if (desc->bInterval <= 14 &&
1701		- dwc->gadget.speed >= USB_SPEED_HIGH) {
	1993	+ dwc->gadget->speed >= USB_SPEED_HIGH) {
1702	1994	u32 frame = __dwc3_gadget_get_frame(dwc);
1703	1995	bool rollover = frame <
1704		- (dep->frame_number & 0x3fff);
	1996	+ (dep->frame_number & DWC3_FRNUMBER_MASK);
1705	1997
1706	1998	/*
1707	1999	* frame_number is set from XferNotReady and may be already
..	..	@@ -1712,54 +2004,46 @@
1712	2004	* rollover has happened since XferNotReady.
1713	2005	*/
1714	2006
1715		- dep->frame_number = (dep->frame_number & ~0x3fff) \|
	2007	+ dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) \|
1716	2008	frame;
1717	2009	if (rollover)
1718	2010	dep->frame_number += BIT(14);
1719	2011	}
1720	2012
1721	2013	for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1722		- dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1);
	2014	+ int future_interval = i + 1;
	2015	+
	2016	+ /* Give the controller at least 500us to schedule transfers */
	2017	+ if (desc->bInterval < 3)
	2018	+ future_interval += 3 - desc->bInterval;
	2019	+
	2020	+ dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1723	2021
1724	2022	ret = __dwc3_gadget_kick_transfer(dep);
1725	2023	if (ret != -EAGAIN)
1726	2024	break;
1727	2025	}
1728	2026
1729		- return ret;
1730		-}
1731		-
1732		-static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep dep, struct dwc3_request req)
1733		-{
1734		- int i;
1735		-
1736	2027	/*
1737		- * If request was already started, this means we had to
1738		- * stop the transfer. With that we also need to ignore
1739		- * all TRBs used by the request, however TRBs can only
1740		- * be modified after completion of END_TRANSFER
1741		- * command. So what we do here is that we wait for
1742		- * END_TRANSFER completion and only after that, we jump
1743		- * over TRBs by clearing HWO and incrementing dequeue
1744		- * pointer.
	2028	+ * After a number of unsuccessful start attempts due to bus-expiry
	2029	+ * status, issue END_TRANSFER command and retry on the next XferNotReady
	2030	+ * event.
1745	2031	*/
1746		- for (i = 0; i < req->num_trbs; i++) {
1747		- struct dwc3_trb *trb;
1748		-
1749		- trb = &dep->trb_pool[dep->trb_dequeue];
1750		- trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1751		- dwc3_ep_inc_deq(dep);
	2032	+ if (ret == -EAGAIN) {
	2033	+ ret = __dwc3_stop_active_transfer(dep, false, true);
	2034	+ if (ret)
	2035	+ dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
1752	2036	}
1753	2037
1754		- req->num_trbs = 0;
	2038	+ return ret;
1755	2039	}
1756	2040
1757	2041	static int __dwc3_gadget_ep_queue(struct dwc3_ep dep, struct dwc3_request req)
1758	2042	{
1759	2043	struct dwc3 *dwc = dep->dwc;
1760	2044
1761		- if (!dep->endpoint.desc) {
1762		- dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
	2045	+ if (!dep->endpoint.desc \|\| !dwc->pullups_connected \|\| !dwc->connected) {
	2046	+ dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1763	2047	dep->name);
1764	2048	return -ESHUTDOWN;
1765	2049	}
..	..	@@ -1770,14 +2054,8 @@
1770	2054
1771	2055	if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1772	2056	"%s: request %pK already in flight\n",
1773		- dep->name, &req->request)) {
1774		- if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1775		- dwc3_gadget_ep_skip_trbs(dep, req);
1776		- req->status = DWC3_REQUEST_STATUS_COMPLETED;
1777		- dwc3_gadget_del_and_unmap_request(dep, req, -EINVAL);
1778		- }
	2057	+ dep->name, &req->request))
1779	2058	return -EINVAL;
1780		- }
1781	2059
1782	2060	pm_runtime_get(dwc->dev);
1783	2061
..	..	@@ -1789,8 +2067,17 @@
1789	2067	list_add_tail(&req->list, &dep->pending_list);
1790	2068	req->status = DWC3_REQUEST_STATUS_QUEUED;
1791	2069
1792		- /* Start the transfer only after the END_TRANSFER is completed */
1793		- if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) {
	2070	+ if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
	2071	+ return 0;
	2072	+
	2073	+ /*
	2074	+ * Start the transfer only after the END_TRANSFER is completed
	2075	+ * and endpoint STALL is cleared.
	2076	+ */
	2077	+ if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) \|\|
	2078	+ (dep->flags & DWC3_EP_WEDGE) \|\|
	2079	+ (dep->flags & DWC3_EP_DELAY_STOP) \|\|
	2080	+ (dep->flags & DWC3_EP_STALL)) {
1794	2081	dep->flags \|= DWC3_EP_DELAY_START;
1795	2082	return 0;
1796	2083	}
..	..	@@ -1804,14 +2091,11 @@
1804	2091	* errors which will force us issue EndTransfer command.
1805	2092	*/
1806	2093	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1807		- if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1808		- !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1809		- return 0;
1810		-
1811		- if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1812		- if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
	2094	+ if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
	2095	+ if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1813	2096	return __dwc3_gadget_start_isoc(dep);
1814		- }
	2097	+
	2098	+ return 0;
1815	2099	}
1816	2100	}
1817	2101
..	..	@@ -1838,14 +2122,64 @@
1838	2122	return ret;
1839	2123	}
1840	2124
	2125	+static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep dep, struct dwc3_request req)
	2126	+{
	2127	+ int i;
	2128	+
	2129	+ /* If req->trb is not set, then the request has not started */
	2130	+ if (!req->trb)
	2131	+ return;
	2132	+
	2133	+ /*
	2134	+ * If request was already started, this means we had to
	2135	+ * stop the transfer. With that we also need to ignore
	2136	+ * all TRBs used by the request, however TRBs can only
	2137	+ * be modified after completion of END_TRANSFER
	2138	+ * command. So what we do here is that we wait for
	2139	+ * END_TRANSFER completion and only after that, we jump
	2140	+ * over TRBs by clearing HWO and incrementing dequeue
	2141	+ * pointer.
	2142	+ */
	2143	+ for (i = 0; i < req->num_trbs; i++) {
	2144	+ struct dwc3_trb *trb;
	2145	+
	2146	+ trb = &dep->trb_pool[dep->trb_dequeue];
	2147	+ trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
	2148	+ dwc3_ep_inc_deq(dep);
	2149	+ }
	2150	+
	2151	+ req->num_trbs = 0;
	2152	+}
	2153	+
1841	2154	static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
1842	2155	{
1843	2156	struct dwc3_request *req;
1844		- struct dwc3_request *tmp;
	2157	+ struct dwc3 *dwc = dep->dwc;
1845	2158
1846		- list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) {
	2159	+ while (!list_empty(&dep->cancelled_list)) {
	2160	+ req = next_request(&dep->cancelled_list);
1847	2161	dwc3_gadget_ep_skip_trbs(dep, req);
1848		- dwc3_gadget_giveback(dep, req, -ECONNRESET);
	2162	+ switch (req->status) {
	2163	+ case DWC3_REQUEST_STATUS_DISCONNECTED:
	2164	+ dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
	2165	+ break;
	2166	+ case DWC3_REQUEST_STATUS_DEQUEUED:
	2167	+ dwc3_gadget_giveback(dep, req, -ECONNRESET);
	2168	+ break;
	2169	+ case DWC3_REQUEST_STATUS_STALLED:
	2170	+ dwc3_gadget_giveback(dep, req, -EPIPE);
	2171	+ break;
	2172	+ default:
	2173	+ dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
	2174	+ dwc3_gadget_giveback(dep, req, -ECONNRESET);
	2175	+ break;
	2176	+ }
	2177	+ /*
	2178	+ * The endpoint is disabled, let the dwc3_remove_requests()
	2179	+ * handle the cleanup.
	2180	+ */
	2181	+ if (!dep->endpoint.desc)
	2182	+ break;
1849	2183	}
1850	2184	}
1851	2185
..	..	@@ -1865,40 +2199,45 @@
1865	2199
1866	2200	spin_lock_irqsave(&dwc->lock, flags);
1867	2201
1868		- list_for_each_entry(r, &dep->pending_list, list) {
	2202	+ list_for_each_entry(r, &dep->cancelled_list, list) {
1869	2203	if (r == req)
1870		- break;
	2204	+ goto out;
1871	2205	}
1872	2206
1873		- if (r != req) {
1874		- list_for_each_entry(r, &dep->started_list, list) {
1875		- if (r == req)
1876		- break;
	2207	+ list_for_each_entry(r, &dep->pending_list, list) {
	2208	+ if (r == req) {
	2209	+ dwc3_gadget_ep_skip_trbs(dep, req);
	2210	+ dwc3_gadget_giveback(dep, req, -ECONNRESET);
	2211	+ goto out;
1877	2212	}
	2213	+ }
	2214	+
	2215	+ list_for_each_entry(r, &dep->started_list, list) {
1878	2216	if (r == req) {
1879	2217	/* wait until it is processed */
1880	2218	dwc3_stop_active_transfer(dep, true, true);
1881	2219
1882		- if (!r->trb)
1883		- goto out0;
	2220	+ /*
	2221	+ * Remove any started request if the transfer is
	2222	+ * cancelled.
	2223	+ */
	2224	+ dwc3_gadget_move_cancelled_request(r, DWC3_REQUEST_STATUS_DEQUEUED);
1884	2225
1885		- dwc3_gadget_move_cancelled_request(req);
1886		- if (dep->flags & DWC3_EP_TRANSFER_STARTED)
1887		- goto out0;
1888		- else
1889		- goto out1;
	2226	+ dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
	2227	+
	2228	+ if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
	2229	+ dwc3_gadget_ep_skip_trbs(dep, req);
	2230	+ dwc3_gadget_giveback(dep, req, -ECONNRESET);
	2231	+ }
	2232	+
	2233	+ goto out;
1890	2234	}
1891		- dev_err(dwc->dev, "request %pK was not queued to %s\n",
1892		- request, ep->name);
1893		- ret = -EINVAL;
1894		- goto out0;
1895	2235	}
1896	2236
1897		-out1:
1898		- dwc3_gadget_ep_skip_trbs(dep, req);
1899		- dwc3_gadget_giveback(dep, req, -ECONNRESET);
1900		-
1901		-out0:
	2237	+ dev_err(dwc->dev, "request %pK was not queued to %s\n",
	2238	+ request, ep->name);
	2239	+ ret = -EINVAL;
	2240	+out:
1902	2241	spin_unlock_irqrestore(&dwc->lock, flags);
1903	2242
1904	2243	return ret;
..	..	@@ -1909,6 +2248,7 @@
1909	2248	struct dwc3_gadget_ep_cmd_params params;
1910	2249	struct dwc3 *dwc = dep->dwc;
1911	2250	int ret;
	2251	+ struct dwc3_vendor *vdwc = container_of(dwc, struct dwc3_vendor, dwc);
1912	2252
1913	2253	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1914	2254	dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
..	..	@@ -1920,8 +2260,8 @@
1920	2260	if (value) {
1921	2261	struct dwc3_trb *trb;
1922	2262
1923		- unsigned transfer_in_flight;
1924		- unsigned started;
	2263	+ unsigned int transfer_in_flight;
	2264	+ unsigned int started;
1925	2265
1926	2266	if (dep->number > 1)
1927	2267	trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
..	..	@@ -1944,13 +2284,44 @@
1944	2284	else
1945	2285	dep->flags \|= DWC3_EP_STALL;
1946	2286	} else {
	2287	+ /*
	2288	+ * Don't issue CLEAR_STALL command to control endpoints. The
	2289	+ * controller automatically clears the STALL when it receives
	2290	+ * the SETUP token.
	2291	+ */
	2292	+ if (dep->number <= 1) {
	2293	+ dep->flags &= ~(DWC3_EP_STALL \| DWC3_EP_WEDGE);
	2294	+ return 0;
	2295	+ }
	2296	+
	2297	+ dwc3_stop_active_transfer(dep, true, true);
	2298	+
	2299	+ if (!list_empty(&dep->started_list))
	2300	+ dep->flags \|= DWC3_EP_DELAY_START;
	2301	+
	2302	+ if (dep->flags & DWC3_EP_END_TRANSFER_PENDING \|\|
	2303	+ (dep->flags & DWC3_EP_DELAY_STOP)) {
	2304	+ dep->flags \|= DWC3_EP_PENDING_CLEAR_STALL;
	2305	+ if (protocol)
	2306	+ vdwc->clear_stall_protocol = dep->number;
	2307	+
	2308	+ return 0;
	2309	+ }
1947	2310
1948	2311	ret = dwc3_send_clear_stall_ep_cmd(dep);
1949		- if (ret)
	2312	+ if (ret) {
1950	2313	dev_err(dwc->dev, "failed to clear STALL on %s\n",
1951	2314	dep->name);
1952		- else
1953		- dep->flags &= ~(DWC3_EP_STALL \| DWC3_EP_WEDGE);
	2315	+ return ret;
	2316	+ }
	2317	+
	2318	+ dep->flags &= ~(DWC3_EP_STALL \| DWC3_EP_WEDGE);
	2319	+
	2320	+ if ((dep->flags & DWC3_EP_DELAY_START) &&
	2321	+ !usb_endpoint_xfer_isoc(dep->endpoint.desc))
	2322	+ __dwc3_gadget_kick_transfer(dep);
	2323	+
	2324	+ dep->flags &= ~DWC3_EP_DELAY_START;
1954	2325	}
1955	2326
1956	2327	return ret;
..	..	@@ -2050,7 +2421,6 @@
2050	2421	link_state = DWC3_DSTS_USBLNKST(reg);
2051	2422
2052	2423	switch (link_state) {
2053		- case DWC3_LINK_STATE_U0:
2054	2424	case DWC3_LINK_STATE_RESET:
2055	2425	case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
2056	2426	case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
..	..	@@ -2062,15 +2432,6 @@
2062	2432	return -EINVAL;
2063	2433	}
2064	2434
2065		- /*
2066		- * dwc3 gadget wakeup from host resume signal
2067		- * when the whole system enter suspend.
2068		- */
2069		- if (link_state == DWC3_LINK_STATE_U0) {
2070		- dwc->link_state = link_state;
2071		- return 0;
2072		- }
2073		-
2074	2435	ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2075	2436	if (ret < 0) {
2076	2437	dev_err(dwc->dev, "failed to put link in Recovery\n");
..	..	@@ -2078,7 +2439,7 @@
2078	2439	}
2079	2440
2080	2441	/* Recent versions do this automatically */
2081		- if (dwc->revision < DWC3_REVISION_194A) {
	2442	+ if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2082	2443	/* write zeroes to Link Change Request */
2083	2444	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2084	2445	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
..	..	@@ -2130,28 +2491,140 @@
2130	2491	return 0;
2131	2492	}
2132	2493
	2494	+static void dwc3_stop_active_transfers(struct dwc3 *dwc)
	2495	+{
	2496	+ u32 epnum;
	2497	+
	2498	+ for (epnum = 2; epnum < dwc->num_eps; epnum++) {
	2499	+ struct dwc3_ep *dep;
	2500	+
	2501	+ dep = dwc->eps[epnum];
	2502	+ if (!dep)
	2503	+ continue;
	2504	+
	2505	+ dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
	2506	+ }
	2507	+}
	2508	+
	2509	+static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
	2510	+{
	2511	+ enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate;
	2512	+ u32 reg;
	2513	+
	2514	+ if (ssp_rate == USB_SSP_GEN_UNKNOWN)
	2515	+ ssp_rate = dwc->max_ssp_rate;
	2516	+
	2517	+ reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	2518	+ reg &= ~DWC3_DCFG_SPEED_MASK;
	2519	+ reg &= ~DWC3_DCFG_NUMLANES(~0);
	2520	+
	2521	+ if (ssp_rate == USB_SSP_GEN_1x2)
	2522	+ reg \|= DWC3_DCFG_SUPERSPEED;
	2523	+ else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
	2524	+ reg \|= DWC3_DCFG_SUPERSPEED_PLUS;
	2525	+
	2526	+ if (ssp_rate != USB_SSP_GEN_2x1 &&
	2527	+ dwc->max_ssp_rate != USB_SSP_GEN_2x1)
	2528	+ reg \|= DWC3_DCFG_NUMLANES(1);
	2529	+
	2530	+ dwc3_writel(dwc->regs, DWC3_DCFG, reg);
	2531	+}
	2532	+
	2533	+static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
	2534	+{
	2535	+ enum usb_device_speed speed;
	2536	+ u32 reg;
	2537	+
	2538	+ speed = dwc->gadget_max_speed;
	2539	+ if (speed == USB_SPEED_UNKNOWN \|\| speed > dwc->maximum_speed)
	2540	+ speed = dwc->maximum_speed;
	2541	+
	2542	+ if (speed == USB_SPEED_SUPER_PLUS &&
	2543	+ DWC3_IP_IS(DWC32)) {
	2544	+ __dwc3_gadget_set_ssp_rate(dwc);
	2545	+ return;
	2546	+ }
	2547	+
	2548	+ reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	2549	+ reg &= ~(DWC3_DCFG_SPEED_MASK);
	2550	+
	2551	+ /*
	2552	+ * WORKAROUND: DWC3 revision < 2.20a have an issue
	2553	+ * which would cause metastability state on Run/Stop
	2554	+ * bit if we try to force the IP to USB2-only mode.
	2555	+ *
	2556	+ * Because of that, we cannot configure the IP to any
	2557	+ * speed other than the SuperSpeed
	2558	+ *
	2559	+ * Refers to:
	2560	+ *
	2561	+ * STAR#9000525659: Clock Domain Crossing on DCTL in
	2562	+ * USB 2.0 Mode
	2563	+ */
	2564	+ if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
	2565	+ !dwc->dis_metastability_quirk) {
	2566	+ reg \|= DWC3_DCFG_SUPERSPEED;
	2567	+ } else {
	2568	+ switch (speed) {
	2569	+ case USB_SPEED_LOW:
	2570	+ reg \|= DWC3_DCFG_LOWSPEED;
	2571	+ break;
	2572	+ case USB_SPEED_FULL:
	2573	+ reg \|= DWC3_DCFG_FULLSPEED;
	2574	+ break;
	2575	+ case USB_SPEED_HIGH:
	2576	+ reg \|= DWC3_DCFG_HIGHSPEED;
	2577	+ break;
	2578	+ case USB_SPEED_SUPER:
	2579	+ reg \|= DWC3_DCFG_SUPERSPEED;
	2580	+ break;
	2581	+ case USB_SPEED_SUPER_PLUS:
	2582	+ if (DWC3_IP_IS(DWC3))
	2583	+ reg \|= DWC3_DCFG_SUPERSPEED;
	2584	+ else
	2585	+ reg \|= DWC3_DCFG_SUPERSPEED_PLUS;
	2586	+ break;
	2587	+ default:
	2588	+ dev_err(dwc->dev, "invalid speed (%d)\n", speed);
	2589	+
	2590	+ if (DWC3_IP_IS(DWC3))
	2591	+ reg \|= DWC3_DCFG_SUPERSPEED;
	2592	+ else
	2593	+ reg \|= DWC3_DCFG_SUPERSPEED_PLUS;
	2594	+ }
	2595	+ }
	2596	+
	2597	+ if (DWC3_IP_IS(DWC32) &&
	2598	+ speed > USB_SPEED_UNKNOWN &&
	2599	+ speed < USB_SPEED_SUPER_PLUS)
	2600	+ reg &= ~DWC3_DCFG_NUMLANES(~0);
	2601	+
	2602	+ dwc3_writel(dwc->regs, DWC3_DCFG, reg);
	2603	+}
	2604	+
2133	2605	static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
2134	2606	{
2135	2607	u32 reg;
2136		- u32 timeout = 500;
	2608	+ u32 timeout = 2000;
2137	2609
2138	2610	if (pm_runtime_suspended(dwc->dev))
2139	2611	return 0;
2140	2612
2141	2613	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2142	2614	if (is_on) {
2143		- if (dwc->revision <= DWC3_REVISION_187A) {
	2615	+ if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2144	2616	reg &= ~DWC3_DCTL_TRGTULST_MASK;
2145	2617	reg \|= DWC3_DCTL_TRGTULST_RX_DET;
2146	2618	}
2147	2619
2148		- if (dwc->revision >= DWC3_REVISION_194A)
	2620	+ if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2149	2621	reg &= ~DWC3_DCTL_KEEP_CONNECT;
2150	2622	reg \|= DWC3_DCTL_RUN_STOP;
2151	2623
2152	2624	if (dwc->has_hibernation)
2153	2625	reg \|= DWC3_DCTL_KEEP_CONNECT;
2154	2626
	2627	+ __dwc3_gadget_set_speed(dwc);
2155	2628	dwc->pullups_connected = true;
2156	2629	} else {
2157	2630	reg &= ~DWC3_DCTL_RUN_STOP;
..	..	@@ -2162,9 +2635,10 @@
2162	2635	dwc->pullups_connected = false;
2163	2636	}
2164	2637
2165		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	2638	+ dwc3_gadget_dctl_write_safe(dwc, reg);
2166	2639
2167	2640	do {
	2641	+ usleep_range(1000, 2000);
2168	2642	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2169	2643	reg &= DWC3_DSTS_DEVCTRLHLT;
2170	2644	} while (--timeout && !(!is_on ^ !reg));
..	..	@@ -2175,42 +2649,125 @@
2175	2649	return 0;
2176	2650	}
2177	2651
2178		-static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2179		-{
2180		- struct dwc3 *dwc = gadget_to_dwc(g);
2181		- unsigned long flags;
2182		- int ret;
	2652	+static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
	2653	+static void __dwc3_gadget_stop(struct dwc3 *dwc);
	2654	+static int __dwc3_gadget_start(struct dwc3 *dwc);
2183	2655
2184		- is_on = !!is_on;
	2656	+static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
	2657	+{
	2658	+ unsigned long flags;
	2659	+
	2660	+ spin_lock_irqsave(&dwc->lock, flags);
	2661	+ dwc->connected = false;
2185	2662
2186	2663	/*
2187	2664	* Per databook, when we want to stop the gadget, if a control transfer
2188	2665	* is still in process, complete it and get the core into setup phase.
2189	2666	*/
2190		- if (!is_on && dwc->ep0state != EP0_SETUP_PHASE &&
	2667	+ if (dwc->ep0state != EP0_SETUP_PHASE &&
2191	2668	dwc->ep0state != EP0_UNCONNECTED) {
	2669	+ int ret;
	2670	+
	2671	+ if (dwc->delayed_status)
	2672	+ dwc3_ep0_send_delayed_status(dwc);
	2673	+
2192	2674	reinit_completion(&dwc->ep0_in_setup);
2193	2675
	2676	+ spin_unlock_irqrestore(&dwc->lock, flags);
2194	2677	ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2195	2678	msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
	2679	+ spin_lock_irqsave(&dwc->lock, flags);
2196	2680	if (ret == 0)
2197	2681	dev_warn(dwc->dev, "timed out waiting for SETUP phase\n");
2198	2682	}
2199	2683
2200		- spin_lock_irqsave(&dwc->lock, flags);
2201		- ret = dwc3_gadget_run_stop(dwc, is_on, false);
	2684	+ /*
	2685	+ * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
	2686	+ * Section 4.1.8 Table 4-7, it states that for a device-initiated
	2687	+ * disconnect, the SW needs to ensure that it sends "a DEPENDXFER
	2688	+ * command for any active transfers" before clearing the RunStop
	2689	+ * bit.
	2690	+ */
	2691	+ dwc3_stop_active_transfers(dwc);
	2692	+ __dwc3_gadget_stop(dwc);
2202	2693	spin_unlock_irqrestore(&dwc->lock, flags);
	2694	+
	2695	+ /*
	2696	+ * Note: if the GEVNTCOUNT indicates events in the event buffer, the
	2697	+ * driver needs to acknowledge them before the controller can halt.
	2698	+ * Simply let the interrupt handler acknowledges and handle the
	2699	+ * remaining event generated by the controller while polling for
	2700	+ * DSTS.DEVCTLHLT.
	2701	+ */
	2702	+ return dwc3_gadget_run_stop(dwc, false, false);
	2703	+}
	2704	+
	2705	+static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
	2706	+{
	2707	+ struct dwc3 *dwc = gadget_to_dwc(g);
	2708	+ struct dwc3_vendor *vdwc = container_of(dwc, struct dwc3_vendor, dwc);
	2709	+ int ret;
	2710	+
	2711	+ is_on = !!is_on;
	2712	+
	2713	+ vdwc->softconnect = is_on;
	2714	+
	2715	+ /*
	2716	+ * Avoid issuing a runtime resume if the device is already in the
	2717	+ * suspended state during gadget disconnect. DWC3 gadget was already
	2718	+ * halted/stopped during runtime suspend.
	2719	+ */
	2720	+ if (!is_on) {
	2721	+ pm_runtime_barrier(dwc->dev);
	2722	+ if (pm_runtime_suspended(dwc->dev))
	2723	+ return 0;
	2724	+ }
	2725	+
	2726	+ /*
	2727	+ * Check the return value for successful resume, or error. For a
	2728	+ * successful resume, the DWC3 runtime PM resume routine will handle
	2729	+ * the run stop sequence, so avoid duplicate operations here.
	2730	+ */
	2731	+ ret = pm_runtime_get_sync(dwc->dev);
	2732	+ if (!ret \|\| ret < 0) {
	2733	+ pm_runtime_put(dwc->dev);
	2734	+ return 0;
	2735	+ }
	2736	+
	2737	+ if (dwc->pullups_connected == is_on) {
	2738	+ pm_runtime_put(dwc->dev);
	2739	+ return 0;
	2740	+ }
	2741	+
	2742	+ synchronize_irq(dwc->irq_gadget);
	2743	+
	2744	+ if (!is_on) {
	2745	+ ret = dwc3_gadget_soft_disconnect(dwc);
	2746	+ } else {
	2747	+ /*
	2748	+ * In the Synopsys DWC_usb31 1.90a programming guide section
	2749	+ * 4.1.9, it specifies that for a reconnect after a
	2750	+ * device-initiated disconnect requires a core soft reset
	2751	+ * (DCTL.CSftRst) before enabling the run/stop bit.
	2752	+ */
	2753	+ dwc3_core_soft_reset(dwc);
	2754	+
	2755	+ dwc3_event_buffers_setup(dwc);
	2756	+ __dwc3_gadget_start(dwc);
	2757	+ ret = dwc3_gadget_run_stop(dwc, true, false);
	2758	+ }
	2759	+
	2760	+ pm_runtime_put(dwc->dev);
2203	2761
2204	2762	return ret;
2205	2763	}
2206	2764
2207		-void dwc3_gadget_enable_irq(struct dwc3 *dwc)
	2765	+static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2208	2766	{
2209	2767	u32 reg;
2210	2768
2211	2769	/* Enable all but Start and End of Frame IRQs */
2212		- reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN \|
2213		- DWC3_DEVTEN_EVNTOVERFLOWEN \|
	2770	+ reg = (DWC3_DEVTEN_EVNTOVERFLOWEN \|
2214	2771	DWC3_DEVTEN_CMDCMPLTEN \|
2215	2772	DWC3_DEVTEN_ERRTICERREN \|
2216	2773	DWC3_DEVTEN_WKUPEVTEN \|
..	..	@@ -2218,17 +2775,17 @@
2218	2775	DWC3_DEVTEN_USBRSTEN \|
2219	2776	DWC3_DEVTEN_DISCONNEVTEN);
2220	2777
2221		- if (dwc->revision < DWC3_REVISION_250A)
	2778	+ if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2222	2779	reg \|= DWC3_DEVTEN_ULSTCNGEN;
2223	2780
2224	2781	/* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2225		- if (dwc->revision >= DWC3_REVISION_230A)
2226		- reg \|= DWC3_DEVTEN_EOPFEN;
	2782	+ if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
	2783	+ reg \|= DWC3_DEVTEN_U3L2L1SUSPEN;
2227	2784
2228	2785	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2229	2786	}
2230	2787
2231		-void dwc3_gadget_disable_irq(struct dwc3 *dwc)
	2788	+static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2232	2789	{
2233	2790	/* mask all interrupts */
2234	2791	dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
..	..	@@ -2266,7 +2823,7 @@
2266	2823	u32 reg;
2267	2824
2268	2825	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2269		- mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
	2826	+ mdwidth = dwc3_mdwidth(dwc);
2270	2827
2271	2828	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2272	2829	nump = min_t(u32, nump, 16);
..	..	@@ -2283,6 +2840,15 @@
2283	2840	struct dwc3_ep *dep;
2284	2841	int ret = 0;
2285	2842	u32 reg;
	2843	+
	2844	+ /*
	2845	+ * If the DWC3 is in runtime suspend, the clocks maybe
	2846	+ * disabled, so avoid enable the DWC3 endpoints here.
	2847	+ * The DWC3 runtime PM resume routine will handle the
	2848	+ * gadget start sequence.
	2849	+ */
	2850	+ if (pm_runtime_suspended(dwc->dev))
	2851	+ return ret;
2286	2852
2287	2853	/*
2288	2854	* Use IMOD if enabled via dwc->imod_interval. Otherwise, if
..	..	@@ -2303,19 +2869,31 @@
2303	2869	* bursts of data without going through any sort of endpoint throttling.
2304	2870	*/
2305	2871	reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2306		- if (dwc3_is_usb31(dwc))
2307		- reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2308		- else
	2872	+ if (DWC3_IP_IS(DWC3))
2309	2873	reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
	2874	+ else
	2875	+ reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2310	2876
2311	2877	dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2312	2878
2313	2879	dwc3_gadget_setup_nump(dwc);
2314	2880
	2881	+ /*
	2882	+ * Currently the controller handles single stream only. So, Ignore
	2883	+ * Packet Pending bit for stream selection and don't search for another
	2884	+ * stream if the host sends Data Packet with PP=0 (for OUT direction) or
	2885	+ * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
	2886	+ * the stream performance.
	2887	+ */
	2888	+ reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	2889	+ reg \|= DWC3_DCFG_IGNSTRMPP;
	2890	+ dwc3_writel(dwc->regs, DWC3_DCFG, reg);
	2891	+
2315	2892	/* Start with SuperSpeed Default */
2316	2893	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2317	2894
2318	2895	dep = dwc->eps[0];
	2896	+ dep->flags = 0;
2319	2897	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2320	2898	if (ret) {
2321	2899	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
..	..	@@ -2323,6 +2901,7 @@
2323	2901	}
2324	2902
2325	2903	dep = dwc->eps[1];
	2904	+ dep->flags = 0;
2326	2905	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2327	2906	if (ret) {
2328	2907	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
..	..	@@ -2331,6 +2910,7 @@
2331	2910
2332	2911	/* begin to receive SETUP packets */
2333	2912	dwc->ep0state = EP0_SETUP_PHASE;
	2913	+ dwc->ep0_bounced = false;
2334	2914	dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2335	2915	dwc->delayed_status = false;
2336	2916	dwc3_ep0_out_start(dwc);
..	..	@@ -2366,17 +2946,13 @@
2366	2946	spin_lock_irqsave(&dwc->lock, flags);
2367	2947	if (dwc->gadget_driver) {
2368	2948	dev_err(dwc->dev, "%s is already bound to %s\n",
2369		- dwc->gadget.name,
	2949	+ dwc->gadget->name,
2370	2950	dwc->gadget_driver->driver.name);
2371	2951	ret = -EBUSY;
2372	2952	goto err1;
2373	2953	}
2374	2954
2375	2955	dwc->gadget_driver = driver;
2376		-
2377		- if (pm_runtime_active(dwc->dev))
2378		- __dwc3_gadget_start(dwc);
2379		-
2380	2956	spin_unlock_irqrestore(&dwc->lock, flags);
2381	2957
2382	2958	return 0;
..	..	@@ -2402,28 +2978,59 @@
2402	2978	unsigned long flags;
2403	2979
2404	2980	spin_lock_irqsave(&dwc->lock, flags);
2405		-
2406	2981	if (!dwc->gadget_driver) {
2407	2982	spin_unlock_irqrestore(&dwc->lock, flags);
2408	2983	dev_warn(dwc->dev, "%s is already stopped\n",
2409		- dwc->gadget.name);
2410		- goto out0;
	2984	+ dwc->gadget->name);
	2985	+ goto out;
2411	2986	}
2412		-
2413		- if (pm_runtime_suspended(dwc->dev))
2414		- goto out1;
2415		-
2416		- __dwc3_gadget_stop(dwc);
2417		-
2418		-out1:
2419		- dwc->fifo_resize_status = false;
2420	2987	dwc->gadget_driver = NULL;
	2988	+ dwc->max_cfg_eps = 0;
2421	2989	spin_unlock_irqrestore(&dwc->lock, flags);
2422	2990
2423	2991	free_irq(dwc->irq_gadget, dwc->ev_buf);
2424	2992
2425		-out0:
	2993	+out:
2426	2994	return 0;
	2995	+}
	2996	+
	2997	+static void dwc3_gadget_config_params(struct usb_gadget *g,
	2998	+ struct usb_dcd_config_params *params)
	2999	+{
	3000	+ struct dwc3 *dwc = gadget_to_dwc(g);
	3001	+
	3002	+ params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
	3003	+ params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
	3004	+
	3005	+ /* Recommended BESL */
	3006	+ if (!dwc->dis_enblslpm_quirk) {
	3007	+ /*
	3008	+ * If the recommended BESL baseline is 0 or if the BESL deep is
	3009	+ * less than 2, Microsoft's Windows 10 host usb stack will issue
	3010	+ * a usb reset immediately after it receives the extended BOS
	3011	+ * descriptor and the enumeration will fail. To maintain
	3012	+ * compatibility with the Windows' usb stack, let's set the
	3013	+ * recommended BESL baseline to 1 and clamp the BESL deep to be
	3014	+ * within 2 to 15.
	3015	+ */
	3016	+ params->besl_baseline = 1;
	3017	+ if (dwc->is_utmi_l1_suspend)
	3018	+ params->besl_deep =
	3019	+ clamp_t(u8, dwc->hird_threshold, 2, 15);
	3020	+ }
	3021	+
	3022	+ /* U1 Device exit Latency */
	3023	+ if (dwc->dis_u1_entry_quirk)
	3024	+ params->bU1devExitLat = 0;
	3025	+ else
	3026	+ params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
	3027	+
	3028	+ /* U2 Device exit Latency */
	3029	+ if (dwc->dis_u2_entry_quirk)
	3030	+ params->bU2DevExitLat = 0;
	3031	+ else
	3032	+ params->bU2DevExitLat =
	3033	+ cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
2427	3034	}
2428	3035
2429	3036	static void dwc3_gadget_set_speed(struct usb_gadget *g,
..	..	@@ -2431,67 +3038,94 @@
2431	3038	{
2432	3039	struct dwc3 *dwc = gadget_to_dwc(g);
2433	3040	unsigned long flags;
2434		- u32 reg;
2435		-
2436		- /*
2437		- * To prevent Android 10 from trying to call UDC and failed constantly
2438		- * while dwc3 is suspended, we let the UDC node always exist.
2439		- * If not return here, it may cause crashes.
2440		- */
2441		- if (pm_runtime_suspended(dwc->dev))
2442		- return;
2443	3041
2444	3042	spin_lock_irqsave(&dwc->lock, flags);
2445		- reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2446		- reg &= ~(DWC3_DCFG_SPEED_MASK);
	3043	+ dwc->gadget_max_speed = speed;
	3044	+ spin_unlock_irqrestore(&dwc->lock, flags);
	3045	+}
2447	3046
2448		- /*
2449		- * WORKAROUND: DWC3 revision < 2.20a have an issue
2450		- * which would cause metastability state on Run/Stop
2451		- * bit if we try to force the IP to USB2-only mode.
2452		- *
2453		- * Because of that, we cannot configure the IP to any
2454		- * speed other than the SuperSpeed
2455		- *
2456		- * Refers to:
2457		- *
2458		- * STAR#9000525659: Clock Domain Crossing on DCTL in
2459		- * USB 2.0 Mode
2460		- */
2461		- if (dwc->revision < DWC3_REVISION_220A &&
2462		- !dwc->dis_metastability_quirk) {
2463		- reg \|= DWC3_DCFG_SUPERSPEED;
2464		- } else {
2465		- switch (speed) {
2466		- case USB_SPEED_LOW:
2467		- reg \|= DWC3_DCFG_LOWSPEED;
2468		- break;
2469		- case USB_SPEED_FULL:
2470		- reg \|= DWC3_DCFG_FULLSPEED;
2471		- break;
2472		- case USB_SPEED_HIGH:
2473		- reg \|= DWC3_DCFG_HIGHSPEED;
2474		- break;
2475		- case USB_SPEED_SUPER:
2476		- reg \|= DWC3_DCFG_SUPERSPEED;
2477		- break;
2478		- case USB_SPEED_SUPER_PLUS:
2479		- if (dwc3_is_usb31(dwc))
2480		- reg \|= DWC3_DCFG_SUPERSPEED_PLUS;
2481		- else
2482		- reg \|= DWC3_DCFG_SUPERSPEED;
2483		- break;
2484		- default:
2485		- dev_err(dwc->dev, "invalid speed (%d)\n", speed);
	3047	+static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
	3048	+ enum usb_ssp_rate rate)
	3049	+{
	3050	+ struct dwc3 *dwc = gadget_to_dwc(g);
	3051	+ unsigned long flags;
2486	3052
2487		- if (dwc->revision & DWC3_REVISION_IS_DWC31)
2488		- reg \|= DWC3_DCFG_SUPERSPEED_PLUS;
2489		- else
2490		- reg \|= DWC3_DCFG_SUPERSPEED;
2491		- }
	3053	+ spin_lock_irqsave(&dwc->lock, flags);
	3054	+ dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
	3055	+ dwc->gadget_ssp_rate = rate;
	3056	+ spin_unlock_irqrestore(&dwc->lock, flags);
	3057	+}
	3058	+
	3059	+static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
	3060	+{
	3061	+ struct dwc3 *dwc = gadget_to_dwc(g);
	3062	+ union power_supply_propval val = {0};
	3063	+ int ret;
	3064	+
	3065	+ if (dwc->usb2_phy)
	3066	+ return usb_phy_set_power(dwc->usb2_phy, mA);
	3067	+
	3068	+ if (!dwc->usb_psy)
	3069	+ return -EOPNOTSUPP;
	3070	+
	3071	+ val.intval = 1000 * mA;
	3072	+ ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
	3073	+
	3074	+ return ret;
	3075	+}
	3076	+
	3077	+/**
	3078	+ * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
	3079	+ * @g: pointer to the USB gadget
	3080	+ *
	3081	+ * Used to record the maximum number of endpoints being used in a USB composite
	3082	+ * device. (across all configurations) This is to be used in the calculation
	3083	+ * of the TXFIFO sizes when resizing internal memory for individual endpoints.
	3084	+ * It will help ensured that the resizing logic reserves enough space for at
	3085	+ * least one max packet.
	3086	+ */
	3087	+static int dwc3_gadget_check_config(struct usb_gadget *g)
	3088	+{
	3089	+ struct dwc3 *dwc = gadget_to_dwc(g);
	3090	+ struct usb_ep *ep;
	3091	+ int fifo_size = 0;
	3092	+ int ram1_depth;
	3093	+ int ep_num = 0;
	3094	+
	3095	+ if (!dwc->do_fifo_resize)
	3096	+ return 0;
	3097	+
	3098	+ list_for_each_entry(ep, &g->ep_list, ep_list) {
	3099	+ /* Only interested in the IN endpoints */
	3100	+ if (ep->claimed && (ep->address & USB_DIR_IN))
	3101	+ ep_num++;
2492	3102	}
2493		- dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2494	3103
	3104	+ if (ep_num <= dwc->max_cfg_eps)
	3105	+ return 0;
	3106	+
	3107	+ /* Update the max number of eps in the composition */
	3108	+ dwc->max_cfg_eps = ep_num;
	3109	+
	3110	+ fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);
	3111	+ /* Based on the equation, increment by one for every ep */
	3112	+ fifo_size += dwc->max_cfg_eps;
	3113	+
	3114	+ /* Check if we can fit a single fifo per endpoint */
	3115	+ ram1_depth = dwc3_gadget_get_tx_fifos_size(dwc);
	3116	+ if (fifo_size > ram1_depth)
	3117	+ return -ENOMEM;
	3118	+
	3119	+ return 0;
	3120	+}
	3121	+
	3122	+static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
	3123	+{
	3124	+ struct dwc3 *dwc = gadget_to_dwc(g);
	3125	+ unsigned long flags;
	3126	+
	3127	+ spin_lock_irqsave(&dwc->lock, flags);
	3128	+ dwc->async_callbacks = enable;
2495	3129	spin_unlock_irqrestore(&dwc->lock, flags);
2496	3130	}
2497	3131
..	..	@@ -2503,6 +3137,11 @@
2503	3137	.udc_start = dwc3_gadget_start,
2504	3138	.udc_stop = dwc3_gadget_stop,
2505	3139	.udc_set_speed = dwc3_gadget_set_speed,
	3140	+ .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate,
	3141	+ .get_config_params = dwc3_gadget_config_params,
	3142	+ .vbus_draw = dwc3_gadget_vbus_draw,
	3143	+ .check_config = dwc3_gadget_check_config,
	3144	+ .udc_async_callbacks = dwc3_gadget_async_callbacks,
2506	3145	};
2507	3146
2508	3147	/* -------------------------------------------------------------------------- */
..	..	@@ -2515,7 +3154,7 @@
2515	3154	dep->endpoint.maxburst = 1;
2516	3155	dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2517	3156	if (!dep->direction)
2518		- dwc->gadget.ep0 = &dep->endpoint;
	3157	+ dwc->gadget->ep0 = &dep->endpoint;
2519	3158
2520	3159	dep->endpoint.caps.type_control = true;
2521	3160
..	..	@@ -2525,48 +3164,62 @@
2525	3164	static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2526	3165	{
2527	3166	struct dwc3 *dwc = dep->dwc;
2528		- int mdwidth;
	3167	+ u32 mdwidth;
2529	3168	int size;
	3169	+ int maxpacket;
2530	3170
2531		- mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
	3171	+ mdwidth = dwc3_mdwidth(dwc);
	3172	+
2532	3173	/* MDWIDTH is represented in bits, we need it in bytes */
2533	3174	mdwidth /= 8;
2534	3175
2535	3176	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2536		- if (dwc3_is_usb31(dwc))
2537		- size = DWC31_GTXFIFOSIZ_TXFDEF(size);
	3177	+ if (DWC3_IP_IS(DWC3))
	3178	+ size = DWC3_GTXFIFOSIZ_TXFDEP(size);
2538	3179	else
2539		- size = DWC3_GTXFIFOSIZ_TXFDEF(size);
2540		-
2541		- /* FIFO Depth is in MDWDITH bytes. Multiply */
2542		- size *= mdwidth;
	3180	+ size = DWC31_GTXFIFOSIZ_TXFDEP(size);
2543	3181
2544	3182	/*
2545		- * To meet performance requirement, a minimum TxFIFO size of 3x
2546		- * MaxPacketSize is recommended for endpoints that support burst and a
2547		- * minimum TxFIFO size of 2x MaxPacketSize for endpoints that don't
2548		- * support burst. Use those numbers and we can calculate the max packet
2549		- * limit as below.
	3183	+ * maxpacket size is determined as part of the following, after assuming
	3184	+ * a mult value of one maxpacket:
	3185	+ * DWC3 revision 280A and prior:
	3186	+ * fifo_size = mult * (max_packet / mdwidth) + 1;
	3187	+ * maxpacket = mdwidth * (fifo_size - 1);
	3188	+ *
	3189	+ * DWC3 revision 290A and onwards:
	3190	+ * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
	3191	+ * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
2550	3192	*/
2551		- if (dwc->maximum_speed >= USB_SPEED_SUPER)
2552		- size /= 3;
	3193	+ if (DWC3_VER_IS_PRIOR(DWC3, 290A))
	3194	+ maxpacket = mdwidth * (size - 1);
2553	3195	else
2554		- size /= 2;
	3196	+ maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
2555	3197
	3198	+
	3199	+ /*
	3200	+ * To meet performance requirement, a minimum TxFIFO size of 2x
	3201	+ * MaxPacketSize is recommended for endpoints that support for
	3202	+ * Rockchip platform with UVC function.
	3203	+ */
	3204	+ if (IS_REACHABLE(CONFIG_ARCH_ROCKCHIP) &&
	3205	+ (dwc->maximum_speed >= USB_SPEED_HIGH))
	3206	+ maxpacket /= 2;
	3207	+
	3208	+ /* Functionally, space for one max packet is sufficient */
	3209	+ size = min_t(int, maxpacket, 1024);
2556	3210	/*
2557	3211	* If enable tx fifos resize, set each in ep maxpacket
2558	3212	* to 1024, it can avoid being dependent on the default
2559	3213	* fifo size, and more flexible use of endpoints.
2560	3214	*/
2561		- if (dwc->needs_fifo_resize)
	3215	+ if (dwc->do_fifo_resize)
2562	3216	size = 1024;
2563		-
2564	3217	usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2565	3218
2566		- dep->endpoint.max_streams = 15;
	3219	+ dep->endpoint.max_streams = 16;
2567	3220	dep->endpoint.ops = &dwc3_gadget_ep_ops;
2568	3221	list_add_tail(&dep->endpoint.ep_list,
2569		- &dwc->gadget.ep_list);
	3222	+ &dwc->gadget->ep_list);
2570	3223	dep->endpoint.caps.type_iso = true;
2571	3224	dep->endpoint.caps.type_bulk = true;
2572	3225	dep->endpoint.caps.type_int = true;
..	..	@@ -2577,20 +3230,20 @@
2577	3230	static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2578	3231	{
2579	3232	struct dwc3 *dwc = dep->dwc;
2580		- int mdwidth;
	3233	+ u32 mdwidth;
2581	3234	int size;
2582	3235
2583		- mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
	3236	+ mdwidth = dwc3_mdwidth(dwc);
2584	3237
2585	3238	/* MDWIDTH is represented in bits, convert to bytes */
2586	3239	mdwidth /= 8;
2587	3240
2588	3241	/* All OUT endpoints share a single RxFIFO space */
2589	3242	size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
2590		- if (dwc3_is_usb31(dwc))
2591		- size = DWC31_GRXFIFOSIZ_RXFDEP(size);
2592		- else
	3243	+ if (DWC3_IP_IS(DWC3))
2593	3244	size = DWC3_GRXFIFOSIZ_RXFDEP(size);
	3245	+ else
	3246	+ size = DWC31_GRXFIFOSIZ_RXFDEP(size);
2594	3247
2595	3248	/* FIFO depth is in MDWDITH bytes */
2596	3249	size *= mdwidth;
..	..	@@ -2610,10 +3263,10 @@
2610	3263	size /= 3;
2611	3264
2612	3265	usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2613		- dep->endpoint.max_streams = 15;
	3266	+ dep->endpoint.max_streams = 16;
2614	3267	dep->endpoint.ops = &dwc3_gadget_ep_ops;
2615	3268	list_add_tail(&dep->endpoint.ep_list,
2616		- &dwc->gadget.ep_list);
	3269	+ &dwc->gadget->ep_list);
2617	3270	dep->endpoint.caps.type_iso = true;
2618	3271	dep->endpoint.caps.type_bulk = true;
2619	3272	dep->endpoint.caps.type_int = true;
..	..	@@ -2627,20 +3280,20 @@
2627	3280	bool direction = epnum & 1;
2628	3281	int ret;
2629	3282	u8 num = epnum >> 1;
2630		- u8 num_in_eps, num_out_eps;
2631		-
2632		- num_in_eps = DWC3_NUM_IN_EPS(&dwc->hwparams);
2633		- num_out_eps = dwc->num_eps - num_in_eps;
	3283	+ u8 num_in_eps, num_out_eps, min_eps;
2634	3284
2635	3285	dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2636	3286	if (!dep)
2637	3287	return -ENOMEM;
2638	3288
	3289	+ num_in_eps = DWC3_NUM_IN_EPS(&dwc->hwparams);
	3290	+ num_out_eps = dwc->num_eps - num_in_eps;
	3291	+ min_eps = min_t(u8, num_in_eps, num_out_eps);
	3292	+
2639	3293	/* reconfig direction and num if num_out_eps != num_in_eps */
2640		- if ((!direction && ((epnum >> 1) + 1) > num_out_eps) \|\|
2641		- (direction && ((epnum >> 1) + 1) > num_in_eps)) {
2642		- direction = !direction;
2643		- num = num + (epnum & 1);
	3294	+ if (num + 1 > min_eps && num_in_eps != num_out_eps) {
	3295	+ num = epnum - min_eps;
	3296	+ direction = num + 1 > num_out_eps ? 1 : 0;
2644	3297	}
2645	3298
2646	3299	dep->dwc = dwc;
..	..	@@ -2659,12 +3312,7 @@
2659	3312	if (!(dep->number > 1)) {
2660	3313	dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2661	3314	dep->endpoint.comp_desc = NULL;
2662		-#ifdef CONFIG_ARCH_ROCKCHIP
2663		- dep->endpoint.transfer_type = USB_ENDPOINT_XFER_CONTROL;
2664		-#endif
2665	3315	}
2666		-
2667		- spin_lock_init(&dep->lock);
2668	3316
2669	3317	if (num == 0)
2670	3318	ret = dwc3_gadget_init_control_endpoint(dep);
..	..	@@ -2692,7 +3340,7 @@
2692	3340	{
2693	3341	u8 epnum;
2694	3342
2695		- INIT_LIST_HEAD(&dwc->gadget.ep_list);
	3343	+ INIT_LIST_HEAD(&dwc->gadget->ep_list);
2696	3344
2697	3345	for (epnum = 0; epnum < total; epnum++) {
2698	3346	int ret;
..	..	@@ -2773,12 +3421,12 @@
2773	3421	}
2774	3422
2775	3423	/*
2776		- * If we're dealing with unaligned size OUT transfer, we will be left
2777		- * with one TRB pending in the ring. We need to manually clear HWO bit
2778		- * from that TRB.
	3424	+ * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
	3425	+ * this TRB points to the bounce buffer address, it's a MPS alignment
	3426	+ * TRB. Don't add it to req->remaining calculation.
2779	3427	*/
2780		-
2781		- if (req->needs_extra_trb && !(trb->ctrl & DWC3_TRB_CTRL_CHN)) {
	3428	+ if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
	3429	+ trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
2782	3430	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2783	3431	return 1;
2784	3432	}
..	..	@@ -2790,6 +3438,10 @@
2790	3438	return 1;
2791	3439
2792	3440	if (event->status & DEPEVT_STATUS_SHORT && !chain)
	3441	+ return 1;
	3442	+
	3443	+ if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&
	3444	+ DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)
2793	3445	return 1;
2794	3446
2795	3447	if ((trb->ctrl & DWC3_TRB_CTRL_IOC) \|\|
..	..	@@ -2845,6 +3497,7 @@
2845	3497	struct dwc3_request *req, int status)
2846	3498	{
2847	3499	struct dwc3 *dwc = dep->dwc;
	3500	+ int request_status;
2848	3501	int ret;
2849	3502
2850	3503	if (req->request.num_mapped_sgs)
..	..	@@ -2856,35 +3509,26 @@
2856	3509
2857	3510	req->request.actual = req->request.length - req->remaining;
2858	3511
2859		- if (!dwc3_gadget_ep_request_completed(req) &&
2860		- !usb_endpoint_xfer_isoc(dep->endpoint.desc))
	3512	+ if (!dwc3_gadget_ep_request_completed(req))
2861	3513	goto out;
2862	3514
2863	3515	if (req->needs_extra_trb) {
2864		- unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
2865		-
2866	3516	ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2867	3517	status);
2868		-
2869		- /* Reclaim MPS padding TRB for ZLP */
2870		- if (!req->direction && req->request.zero && req->request.length &&
2871		- !usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2872		- (IS_ALIGNED(req->request.length, maxp)))
2873		- ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, status);
2874		-
2875	3518	req->needs_extra_trb = false;
2876	3519	}
2877	3520
2878		- if (event->status & DEPEVT_STATUS_MISSED_ISOC &&
2879		- usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2880		- /*
2881		- * unmap isoc request and move the request
2882		- * to the pending list to wait for kicking
2883		- * transfer again.
2884		- */
	3521	+ /*
	3522	+ * If MISS ISOC happens, we need to move the req from started_list
	3523	+ * to cancelled_list, then unmap the req and clear the HWO of trb.
	3524	+ * Later in the dwc3_gadget_endpoint_trbs_complete(), it will move
	3525	+ * the req from the cancelled_list to the pending_list, and restart
	3526	+ * the req for isoc transfer.
	3527	+ */
	3528	+ if (status == -EXDEV && usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2885	3529	req->remaining = 0;
2886	3530	req->needs_extra_trb = false;
2887		- dwc3_gadget_move_cancelled_request(req);
	3531	+ dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
2888	3532	if (req->trb) {
2889	3533	usb_gadget_unmap_request_by_dev(dwc->sysdev,
2890	3534	&req->request,
..	..	@@ -2893,11 +3537,38 @@
2893	3537	req->trb = NULL;
2894	3538	}
2895	3539	ret = 0;
2896		-
2897	3540	goto out;
2898	3541	}
2899	3542
2900		- dwc3_gadget_giveback(dep, req, status);
	3543	+ /*
	3544	+ * The event status only reflects the status of the TRB with IOC set.
	3545	+ * For the requests that don't set interrupt on completion, the driver
	3546	+ * needs to check and return the status of the completed TRBs associated
	3547	+ * with the request. Use the status of the last TRB of the request.
	3548	+ */
	3549	+ if (req->request.no_interrupt) {
	3550	+ struct dwc3_trb *trb;
	3551	+
	3552	+ trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
	3553	+ switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
	3554	+ case DWC3_TRBSTS_MISSED_ISOC:
	3555	+ /* Isoc endpoint only */
	3556	+ request_status = -EXDEV;
	3557	+ break;
	3558	+ case DWC3_TRB_STS_XFER_IN_PROG:
	3559	+ /* Applicable when End Transfer with ForceRM=0 */
	3560	+ case DWC3_TRBSTS_SETUP_PENDING:
	3561	+ /* Control endpoint only */
	3562	+ case DWC3_TRBSTS_OK:
	3563	+ default:
	3564	+ request_status = 0;
	3565	+ break;
	3566	+ }
	3567	+ } else {
	3568	+ request_status = status;
	3569	+ }
	3570	+
	3571	+ dwc3_gadget_giveback(dep, req, request_status);
2901	3572
2902	3573	out:
2903	3574	return ret;
..	..	@@ -2907,14 +3578,20 @@
2907	3578	const struct dwc3_event_depevt *event, int status)
2908	3579	{
2909	3580	struct dwc3_request *req;
2910		- struct dwc3_request *tmp;
2911	3581
2912		- list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
	3582	+ while (!list_empty(&dep->started_list)) {
2913	3583	int ret;
2914	3584
	3585	+ req = next_request(&dep->started_list);
2915	3586	ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2916	3587	req, status);
2917	3588	if (ret)
	3589	+ break;
	3590	+ /*
	3591	+ * The endpoint is disabled, let the dwc3_remove_requests()
	3592	+ * handle the cleanup.
	3593	+ */
	3594	+ if (!dep->endpoint.desc)
2918	3595	break;
2919	3596	}
2920	3597	}
..	..	@@ -2922,6 +3599,11 @@
2922	3599	static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
2923	3600	{
2924	3601	struct dwc3_request *req;
	3602	+ struct dwc3 *dwc = dep->dwc;
	3603	+
	3604	+ if (!dep->endpoint.desc \|\| !dwc->pullups_connected \|\|
	3605	+ !dwc->connected)
	3606	+ return false;
2925	3607
2926	3608	if (!list_empty(&dep->pending_list))
2927	3609	return true;
..	..	@@ -2943,49 +3625,58 @@
2943	3625	dep->frame_number = event->parameters;
2944	3626	}
2945	3627
2946		-static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2947		- const struct dwc3_event_depevt *event)
	3628	+static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
	3629	+ const struct dwc3_event_depevt *event, int status)
2948	3630	{
2949	3631	struct dwc3 *dwc = dep->dwc;
2950		- unsigned status = 0;
2951		- struct dwc3_request *req;
2952		- struct dwc3_request *tmp;
2953		-
2954		- dwc3_gadget_endpoint_frame_from_event(dep, event);
2955		-
2956		- if (event->status & DEPEVT_STATUS_BUSERR)
2957		- status = -ECONNRESET;
2958		-
2959		- if (event->status & DEPEVT_STATUS_MISSED_ISOC)
2960		- status = -EXDEV;
	3632	+ struct dwc3_request req, tmp;
	3633	+ bool no_started_trb = true;
2961	3634
2962	3635	dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2963	3636
2964		- if (event->status & DEPEVT_STATUS_MISSED_ISOC &&
	3637	+ if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
	3638	+ goto out;
	3639	+
	3640	+ if (!dep->endpoint.desc)
	3641	+ return no_started_trb;
	3642	+
	3643	+ /*
	3644	+ * If MISS ISOC happens, we need to do the following three steps
	3645	+ * to restart the reqs in the cancelled_list and pending_list
	3646	+ * in order.
	3647	+ * Step1. Move all the reqs from pending_list to the tail of
	3648	+ * cancelled_list.
	3649	+ * Step2. Move all the reqs from cancelled_list to the tail
	3650	+ * of pending_list.
	3651	+ * Step3. Stop and restart an isoc transfer.
	3652	+ */
	3653	+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && status == -EXDEV &&
2965	3654	!list_empty(&dep->cancelled_list) &&
2966	3655	!list_empty(&dep->pending_list)) {
2967	3656	list_for_each_entry_safe(req, tmp, &dep->pending_list, list)
2968		- dwc3_gadget_move_cancelled_request(req);
	3657	+ dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
2969	3658	}
2970	3659
2971		- if (event->status & DEPEVT_STATUS_MISSED_ISOC &&
	3660	+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && status == -EXDEV &&
2972	3661	!list_empty(&dep->cancelled_list)) {
2973	3662	list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list)
2974	3663	dwc3_gadget_move_queued_request(req);
2975	3664	}
2976	3665
2977		- if (event->status & DEPEVT_STATUS_MISSED_ISOC &&
2978		- list_empty(&dep->started_list))
	3666	+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
	3667	+ list_empty(&dep->started_list) &&
	3668	+ (list_empty(&dep->pending_list) \|\| status == -EXDEV))
2979	3669	dwc3_stop_active_transfer(dep, true, true);
2980		- else if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2981		- dwc3_gadget_ep_should_continue(dep))
2982		- __dwc3_gadget_kick_transfer(dep);
	3670	+ else if (dwc3_gadget_ep_should_continue(dep))
	3671	+ if (__dwc3_gadget_kick_transfer(dep) == 0)
	3672	+ no_started_trb = false;
2983	3673
	3674	+out:
2984	3675	/*
2985	3676	* WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2986	3677	* See dwc3_gadget_linksts_change_interrupt() for 1st half.
2987	3678	*/
2988		- if (dwc->revision < DWC3_REVISION_183A) {
	3679	+ if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
2989	3680	u32 reg;
2990	3681	int i;
2991	3682
..	..	@@ -2996,7 +3687,7 @@
2996	3687	continue;
2997	3688
2998	3689	if (!list_empty(&dep->started_list))
2999		- return;
	3690	+ return no_started_trb;
3000	3691	}
3001	3692
3002	3693	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
..	..	@@ -3005,13 +3696,171 @@
3005	3696
3006	3697	dwc->u1u2 = 0;
3007	3698	}
	3699	+
	3700	+ return no_started_trb;
	3701	+}
	3702	+
	3703	+static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
	3704	+ const struct dwc3_event_depevt *event)
	3705	+{
	3706	+ int status = 0;
	3707	+
	3708	+ if (!dep->endpoint.desc)
	3709	+ return;
	3710	+
	3711	+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
	3712	+ dwc3_gadget_endpoint_frame_from_event(dep, event);
	3713	+
	3714	+ if (event->status & DEPEVT_STATUS_BUSERR)
	3715	+ status = -ECONNRESET;
	3716	+
	3717	+ if (event->status & DEPEVT_STATUS_MISSED_ISOC)
	3718	+ status = -EXDEV;
	3719	+
	3720	+ dwc3_gadget_endpoint_trbs_complete(dep, event, status);
	3721	+}
	3722	+
	3723	+static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
	3724	+ const struct dwc3_event_depevt *event)
	3725	+{
	3726	+ int status = 0;
	3727	+
	3728	+ dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
	3729	+
	3730	+ if (event->status & DEPEVT_STATUS_BUSERR)
	3731	+ status = -ECONNRESET;
	3732	+
	3733	+ if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
	3734	+ dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3008	3735	}
3009	3736
3010	3737	static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3011	3738	const struct dwc3_event_depevt *event)
3012	3739	{
3013	3740	dwc3_gadget_endpoint_frame_from_event(dep, event);
	3741	+
	3742	+ /*
	3743	+ * The XferNotReady event is generated only once before the endpoint
	3744	+ * starts. It will be generated again when END_TRANSFER command is
	3745	+ * issued. For some controller versions, the XferNotReady event may be
	3746	+ * generated while the END_TRANSFER command is still in process. Ignore
	3747	+ * it and wait for the next XferNotReady event after the command is
	3748	+ * completed.
	3749	+ */
	3750	+ if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
	3751	+ return;
	3752	+
3014	3753	(void) __dwc3_gadget_start_isoc(dep);
	3754	+}
	3755	+
	3756	+static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
	3757	+ const struct dwc3_event_depevt *event)
	3758	+{
	3759	+ u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
	3760	+
	3761	+ if (cmd != DWC3_DEPCMD_ENDTRANSFER)
	3762	+ return;
	3763	+
	3764	+ /*
	3765	+ * The END_TRANSFER command will cause the controller to generate a
	3766	+ * NoStream Event, and it's not due to the host DP NoStream rejection.
	3767	+ * Ignore the next NoStream event.
	3768	+ */
	3769	+ if (dep->stream_capable)
	3770	+ dep->flags \|= DWC3_EP_IGNORE_NEXT_NOSTREAM;
	3771	+
	3772	+ dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
	3773	+ dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
	3774	+ dwc3_gadget_ep_cleanup_cancelled_requests(dep);
	3775	+
	3776	+ if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
	3777	+ struct dwc3 *dwc = dep->dwc;
	3778	+ struct dwc3_vendor *vdwc = container_of(dwc, struct dwc3_vendor, dwc);
	3779	+
	3780	+ dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
	3781	+ if (dwc3_send_clear_stall_ep_cmd(dep)) {
	3782	+ struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
	3783	+
	3784	+ dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
	3785	+ if (dwc->delayed_status)
	3786	+ __dwc3_gadget_ep0_set_halt(ep0, 1);
	3787	+ return;
	3788	+ }
	3789	+
	3790	+ dep->flags &= ~(DWC3_EP_STALL \| DWC3_EP_WEDGE);
	3791	+ if (vdwc->clear_stall_protocol == dep->number)
	3792	+ dwc3_ep0_send_delayed_status(dwc);
	3793	+ }
	3794	+
	3795	+ if ((dep->flags & DWC3_EP_DELAY_START) &&
	3796	+ !usb_endpoint_xfer_isoc(dep->endpoint.desc))
	3797	+ __dwc3_gadget_kick_transfer(dep);
	3798	+
	3799	+ dep->flags &= ~DWC3_EP_DELAY_START;
	3800	+}
	3801	+
	3802	+static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
	3803	+ const struct dwc3_event_depevt *event)
	3804	+{
	3805	+ struct dwc3 *dwc = dep->dwc;
	3806	+
	3807	+ if (event->status == DEPEVT_STREAMEVT_FOUND) {
	3808	+ dep->flags \|= DWC3_EP_FIRST_STREAM_PRIMED;
	3809	+ goto out;
	3810	+ }
	3811	+
	3812	+ /* Note: NoStream rejection event param value is 0 and not 0xFFFF */
	3813	+ switch (event->parameters) {
	3814	+ case DEPEVT_STREAM_PRIME:
	3815	+ /*
	3816	+ * If the host can properly transition the endpoint state from
	3817	+ * idle to prime after a NoStream rejection, there's no need to
	3818	+ * force restarting the endpoint to reinitiate the stream. To
	3819	+ * simplify the check, assume the host follows the USB spec if
	3820	+ * it primed the endpoint more than once.
	3821	+ */
	3822	+ if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
	3823	+ if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
	3824	+ dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
	3825	+ else
	3826	+ dep->flags \|= DWC3_EP_FIRST_STREAM_PRIMED;
	3827	+ }
	3828	+
	3829	+ break;
	3830	+ case DEPEVT_STREAM_NOSTREAM:
	3831	+ if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) \|\|
	3832	+ !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) \|\|
	3833	+ !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))
	3834	+ break;
	3835	+
	3836	+ /*
	3837	+ * If the host rejects a stream due to no active stream, by the
	3838	+ * USB and xHCI spec, the endpoint will be put back to idle
	3839	+ * state. When the host is ready (buffer added/updated), it will
	3840	+ * prime the endpoint to inform the usb device controller. This
	3841	+ * triggers the device controller to issue ERDY to restart the
	3842	+ * stream. However, some hosts don't follow this and keep the
	3843	+ * endpoint in the idle state. No prime will come despite host
	3844	+ * streams are updated, and the device controller will not be
	3845	+ * triggered to generate ERDY to move the next stream data. To
	3846	+ * workaround this and maintain compatibility with various
	3847	+ * hosts, force to reinitate the stream until the host is ready
	3848	+ * instead of waiting for the host to prime the endpoint.
	3849	+ */
	3850	+ if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
	3851	+ unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
	3852	+
	3853	+ dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
	3854	+ } else {
	3855	+ dep->flags \|= DWC3_EP_DELAY_START;
	3856	+ dwc3_stop_active_transfer(dep, true, true);
	3857	+ return;
	3858	+ }
	3859	+ break;
	3860	+ }
	3861	+
	3862	+out:
	3863	+ dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3015	3864	}
3016	3865
3017	3866	static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
..	..	@@ -3019,25 +3868,20 @@
3019	3868	{
3020	3869	struct dwc3_ep *dep;
3021	3870	u8 epnum = event->endpoint_number;
3022		- u8 cmd;
3023	3871
3024	3872	dep = dwc->eps[epnum];
3025	3873
3026	3874	if (!(dep->flags & DWC3_EP_ENABLED)) {
3027		- if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
	3875	+ if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3028	3876	return;
3029	3877
3030	3878	/* Handle only EPCMDCMPLT when EP disabled */
3031		- if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
	3879	+ if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&
	3880	+ !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))
3032	3881	return;
3033	3882	}
3034	3883
3035	3884	if (epnum == 0 \|\| epnum == 1) {
3036		- if (!dwc->connected &&
3037		- event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE) {
3038		- reinit_completion(&dwc->discon_done);
3039		- dwc->connected = true;
3040		- }
3041	3885	dwc3_ep0_interrupt(dwc, event);
3042	3886	return;
3043	3887	}
..	..	@@ -3050,21 +3894,14 @@
3050	3894	dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3051	3895	break;
3052	3896	case DWC3_DEPEVT_EPCMDCMPLT:
3053		- cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3054		-
3055		- if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
3056		- dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3057		- dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3058		- dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3059		- if ((dep->flags & DWC3_EP_DELAY_START) &&
3060		- !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3061		- __dwc3_gadget_kick_transfer(dep);
3062		-
3063		- dep->flags &= ~DWC3_EP_DELAY_START;
3064		- }
	3897	+ dwc3_gadget_endpoint_command_complete(dep, event);
	3898	+ break;
	3899	+ case DWC3_DEPEVT_XFERCOMPLETE:
	3900	+ dwc3_gadget_endpoint_transfer_complete(dep, event);
3065	3901	break;
3066	3902	case DWC3_DEPEVT_STREAMEVT:
3067		- case DWC3_DEPEVT_XFERCOMPLETE:
	3903	+ dwc3_gadget_endpoint_stream_event(dep, event);
	3904	+ break;
3068	3905	case DWC3_DEPEVT_RXTXFIFOEVT:
3069	3906	break;
3070	3907	}
..	..	@@ -3072,27 +3909,27 @@
3072	3909
3073	3910	static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3074	3911	{
3075		- if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
	3912	+ if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3076	3913	spin_unlock(&dwc->lock);
3077		- dwc->gadget_driver->disconnect(&dwc->gadget);
	3914	+ dwc->gadget_driver->disconnect(dwc->gadget);
3078	3915	spin_lock(&dwc->lock);
3079	3916	}
3080	3917	}
3081	3918
3082	3919	static void dwc3_suspend_gadget(struct dwc3 *dwc)
3083	3920	{
3084		- if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
	3921	+ if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3085	3922	spin_unlock(&dwc->lock);
3086		- dwc->gadget_driver->suspend(&dwc->gadget);
	3923	+ dwc->gadget_driver->suspend(dwc->gadget);
3087	3924	spin_lock(&dwc->lock);
3088	3925	}
3089	3926	}
3090	3927
3091	3928	static void dwc3_resume_gadget(struct dwc3 *dwc)
3092	3929	{
3093		- if (dwc->gadget_driver && dwc->gadget_driver->resume) {
	3930	+ if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3094	3931	spin_unlock(&dwc->lock);
3095		- dwc->gadget_driver->resume(&dwc->gadget);
	3932	+ dwc->gadget_driver->resume(dwc->gadget);
3096	3933	spin_lock(&dwc->lock);
3097	3934	}
3098	3935	}
..	..	@@ -3102,23 +3939,43 @@
3102	3939	if (!dwc->gadget_driver)
3103	3940	return;
3104	3941
3105		- if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
	3942	+ if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3106	3943	spin_unlock(&dwc->lock);
3107		- usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
	3944	+ usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3108	3945	spin_lock(&dwc->lock);
3109	3946	}
3110	3947	}
3111	3948
3112		-static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
	3949	+void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3113	3950	bool interrupt)
3114	3951	{
3115		- struct dwc3_gadget_ep_cmd_params params;
3116		- u32 cmd;
3117		- int ret;
	3952	+ struct dwc3 *dwc = dep->dwc;
	3953	+
	3954	+ /*
	3955	+ * Only issue End Transfer command to the control endpoint of a started
	3956	+ * Data Phase. Typically we should only do so in error cases such as
	3957	+ * invalid/unexpected direction as described in the control transfer
	3958	+ * flow of the programming guide.
	3959	+ */
	3960	+ if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
	3961	+ return;
3118	3962
3119	3963	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) \|\|
	3964	+ (dep->flags & DWC3_EP_DELAY_STOP) \|\|
3120	3965	(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3121	3966	return;
	3967	+
	3968	+ /*
	3969	+ * If a Setup packet is received but yet to DMA out, the controller will
	3970	+ * not process the End Transfer command of any endpoint. Polling of its
	3971	+ * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
	3972	+ * timeout. Delay issuing the End Transfer command until the Setup TRB is
	3973	+ * prepared.
	3974	+ */
	3975	+ if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
	3976	+ dep->flags \|= DWC3_EP_DELAY_STOP;
	3977	+ return;
	3978	+ }
3122	3979
3123	3980	/*
3124	3981	* NOTICE: We are violating what the Databook says about the
..	..	@@ -3147,20 +4004,9 @@
3147	4004	* This mode is NOT available on the DWC_usb31 IP.
3148	4005	*/
3149	4006
3150		- cmd = DWC3_DEPCMD_ENDTRANSFER;
3151		- cmd \|= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
3152		- cmd \|= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
3153		- cmd \|= DWC3_DEPCMD_PARAM(dep->resource_index);
3154		- memset(&params, 0, sizeof(params));
3155		- ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
3156		- WARN_ON_ONCE(ret);
3157		- dep->resource_index = 0;
3158		-
3159		- if (!interrupt)
3160		- dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3161		- else
3162		- dep->flags \|= DWC3_EP_END_TRANSFER_PENDING;
	4007	+ __dwc3_stop_active_transfer(dep, force, interrupt);
3163	4008	}
	4009	+EXPORT_SYMBOL_GPL(dwc3_stop_active_transfer);
3164	4010
3165	4011	static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3166	4012	{
..	..	@@ -3188,30 +4034,36 @@
3188	4034	{
3189	4035	int reg;
3190	4036
	4037	+ dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
	4038	+
3191	4039	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3192	4040	reg &= ~DWC3_DCTL_INITU1ENA;
3193		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3194		-
3195	4041	reg &= ~DWC3_DCTL_INITU2ENA;
3196		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	4042	+ dwc3_gadget_dctl_write_safe(dwc, reg);
	4043	+
	4044	+ dwc->connected = false;
3197	4045
3198	4046	dwc3_disconnect_gadget(dwc);
3199	4047
3200		- dwc->gadget.speed = USB_SPEED_UNKNOWN;
	4048	+ dwc->gadget->speed = USB_SPEED_UNKNOWN;
3201	4049	dwc->setup_packet_pending = false;
3202		- usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
	4050	+ usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3203	4051
3204		- dwc->connected = false;
3205		- complete(&dwc->discon_done);
	4052	+ if (dwc->ep0state != EP0_SETUP_PHASE) {
	4053	+ unsigned int dir;
	4054	+
	4055	+ dir = !!dwc->ep0_expect_in;
	4056	+ if (dwc->ep0state == EP0_DATA_PHASE)
	4057	+ dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
	4058	+ else
	4059	+ dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
	4060	+ dwc3_ep0_stall_and_restart(dwc);
	4061	+ }
3206	4062	}
3207	4063
3208	4064	static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3209	4065	{
3210	4066	u32 reg;
3211		-
3212		- if (of_device_is_compatible(dwc->dev->parent->of_node,
3213		- "rockchip,rk3399-dwc3"))
3214		- phy_calibrate(dwc->usb2_generic_phy);
3215	4067
3216	4068	/*
3217	4069	* Ideally, dwc3_reset_gadget() would trigger the function
..	..	@@ -3248,16 +4100,45 @@
3248	4100	* STAR#9000466709: RTL: Device : Disconnect event not
3249	4101	* generated if setup packet pending in FIFO
3250	4102	*/
3251		- if (dwc->revision < DWC3_REVISION_188A) {
	4103	+ if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
3252	4104	if (dwc->setup_packet_pending)
3253	4105	dwc3_gadget_disconnect_interrupt(dwc);
3254	4106	}
3255	4107
3256	4108	dwc3_reset_gadget(dwc);
3257	4109
	4110	+ /*
	4111	+ * From SNPS databook section 8.1.2, the EP0 should be in setup
	4112	+ * phase. So ensure that EP0 is in setup phase by issuing a stall
	4113	+ * and restart if EP0 is not in setup phase.
	4114	+ */
	4115	+ if (dwc->ep0state != EP0_SETUP_PHASE) {
	4116	+ unsigned int dir;
	4117	+
	4118	+ dir = !!dwc->ep0_expect_in;
	4119	+ if (dwc->ep0state == EP0_DATA_PHASE)
	4120	+ dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
	4121	+ else
	4122	+ dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
	4123	+
	4124	+ dwc->eps[0]->trb_enqueue = 0;
	4125	+ dwc->eps[1]->trb_enqueue = 0;
	4126	+
	4127	+ dwc3_ep0_stall_and_restart(dwc);
	4128	+ }
	4129	+
	4130	+ /*
	4131	+ * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
	4132	+ * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
	4133	+ * needs to ensure that it sends "a DEPENDXFER command for any active
	4134	+ * transfers."
	4135	+ */
	4136	+ dwc3_stop_active_transfers(dwc);
	4137	+ dwc->connected = true;
	4138	+
3258	4139	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3259	4140	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
3260		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	4141	+ dwc3_gadget_dctl_write_safe(dwc, reg);
3261	4142	dwc->test_mode = false;
3262	4143	dwc3_clear_stall_all_ep(dwc);
3263	4144
..	..	@@ -3272,11 +4153,21 @@
3272	4153	struct dwc3_ep *dep;
3273	4154	int ret;
3274	4155	u32 reg;
	4156	+ u8 lanes = 1;
3275	4157	u8 speed;
	4158	+ struct dwc3_vendor *vdwc = container_of(dwc, struct dwc3_vendor, dwc);
	4159	+
	4160	+ if (!vdwc->softconnect)
	4161	+ return;
3276	4162
3277	4163	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
3278	4164	speed = reg & DWC3_DSTS_CONNECTSPD;
3279	4165	dwc->speed = speed;
	4166	+
	4167	+ if (DWC3_IP_IS(DWC32))
	4168	+ lanes = DWC3_DSTS_CONNLANES(reg) + 1;
	4169	+
	4170	+ dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
3280	4171
3281	4172	/*
3282	4173	* RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
..	..	@@ -3290,8 +4181,13 @@
3290	4181	switch (speed) {
3291	4182	case DWC3_DSTS_SUPERSPEED_PLUS:
3292	4183	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3293		- dwc->gadget.ep0->maxpacket = 512;
3294		- dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
	4184	+ dwc->gadget->ep0->maxpacket = 512;
	4185	+ dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
	4186	+
	4187	+ if (lanes > 1)
	4188	+ dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
	4189	+ else
	4190	+ dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
3295	4191	break;
3296	4192	case DWC3_DSTS_SUPERSPEED:
3297	4193	/*
..	..	@@ -3307,35 +4203,41 @@
3307	4203	* STAR#9000483510: RTL: SS : USB3 reset event may
3308	4204	* not be generated always when the link enters poll
3309	4205	*/
3310		- if (dwc->revision < DWC3_REVISION_190A)
	4206	+ if (DWC3_VER_IS_PRIOR(DWC3, 190A))
3311	4207	dwc3_gadget_reset_interrupt(dwc);
3312	4208
3313	4209	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3314		- dwc->gadget.ep0->maxpacket = 512;
3315		- dwc->gadget.speed = USB_SPEED_SUPER;
	4210	+ dwc->gadget->ep0->maxpacket = 512;
	4211	+ dwc->gadget->speed = USB_SPEED_SUPER;
	4212	+
	4213	+ if (lanes > 1) {
	4214	+ dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
	4215	+ dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
	4216	+ }
3316	4217	break;
3317	4218	case DWC3_DSTS_HIGHSPEED:
3318	4219	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3319		- dwc->gadget.ep0->maxpacket = 64;
3320		- dwc->gadget.speed = USB_SPEED_HIGH;
	4220	+ dwc->gadget->ep0->maxpacket = 64;
	4221	+ dwc->gadget->speed = USB_SPEED_HIGH;
3321	4222	break;
3322	4223	case DWC3_DSTS_FULLSPEED:
3323	4224	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3324		- dwc->gadget.ep0->maxpacket = 64;
3325		- dwc->gadget.speed = USB_SPEED_FULL;
	4225	+ dwc->gadget->ep0->maxpacket = 64;
	4226	+ dwc->gadget->speed = USB_SPEED_FULL;
3326	4227	break;
3327	4228	case DWC3_DSTS_LOWSPEED:
3328	4229	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
3329		- dwc->gadget.ep0->maxpacket = 8;
3330		- dwc->gadget.speed = USB_SPEED_LOW;
	4230	+ dwc->gadget->ep0->maxpacket = 8;
	4231	+ dwc->gadget->speed = USB_SPEED_LOW;
3331	4232	break;
3332	4233	}
3333	4234
3334		- dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
	4235	+ dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
3335	4236
3336	4237	/* Enable USB2 LPM Capability */
3337	4238
3338		- if ((dwc->revision > DWC3_REVISION_194A) &&
	4239	+ if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
	4240	+ !dwc->usb2_gadget_lpm_disable &&
3339	4241	(speed != DWC3_DSTS_SUPERSPEED) &&
3340	4242	(speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
3341	4243	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
..	..	@@ -3345,7 +4247,8 @@
3345	4247	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3346	4248	reg &= ~(DWC3_DCTL_HIRD_THRES_MASK \| DWC3_DCTL_L1_HIBER_EN);
3347	4249
3348		- reg \|= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
	4250	+ reg \|= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold \|
	4251	+ (dwc->is_utmi_l1_suspend << 4));
3349	4252
3350	4253	/*
3351	4254	* When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
..	..	@@ -3353,23 +4256,24 @@
3353	4256	* BESL value in the LPM token is less than or equal to LPM
3354	4257	* NYET threshold.
3355	4258	*/
3356		- WARN_ONCE(dwc->revision < DWC3_REVISION_240A
3357		- && dwc->has_lpm_erratum,
	4259	+ WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
3358	4260	"LPM Erratum not available on dwc3 revisions < 2.40a\n");
3359	4261
3360		- if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
	4262	+ if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
3361	4263	reg \|= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
3362	4264
3363		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	4265	+ dwc3_gadget_dctl_write_safe(dwc, reg);
3364	4266	} else {
	4267	+ if (dwc->usb2_gadget_lpm_disable) {
	4268	+ reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	4269	+ reg &= ~DWC3_DCFG_LPM_CAP;
	4270	+ dwc3_writel(dwc->regs, DWC3_DCFG, reg);
	4271	+ }
	4272	+
3365	4273	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3366	4274	reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
3367		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	4275	+ dwc3_gadget_dctl_write_safe(dwc, reg);
3368	4276	}
3369		-
3370		-#ifdef CONFIG_ARCH_ROCKCHIP
3371		- dwc3_gadget_resize_tx_fifos(dwc);
3372		-#endif
3373	4277
3374	4278	dep = dwc->eps[0];
3375	4279	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
..	..	@@ -3394,21 +4298,18 @@
3394	4298	*/
3395	4299	}
3396	4300
3397		-static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
	4301	+static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
3398	4302	{
3399		- enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3400	4303	/*
3401	4304	* TODO take core out of low power mode when that's
3402	4305	* implemented.
3403	4306	*/
3404	4307
3405		- if (dwc->gadget_driver && dwc->gadget_driver->resume && dwc->uwk_en) {
	4308	+ if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3406	4309	spin_unlock(&dwc->lock);
3407		- dwc->gadget_driver->resume(&dwc->gadget);
	4310	+ dwc->gadget_driver->resume(dwc->gadget);
3408	4311	spin_lock(&dwc->lock);
3409	4312	}
3410		-
3411		- dwc->link_state = next;
3412	4313	}
3413	4314
3414	4315	static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
..	..	@@ -3435,7 +4336,7 @@
3435	4336	* operational mode
3436	4337	*/
3437	4338	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
3438		- if ((dwc->revision < DWC3_REVISION_250A) &&
	4339	+ if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
3439	4340	(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
3440	4341	if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
3441	4342	(next == DWC3_LINK_STATE_RESUME)) {
..	..	@@ -3461,7 +4362,7 @@
3461	4362	* STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
3462	4363	* core send LGO_Ux entering U0
3463	4364	*/
3464		- if (dwc->revision < DWC3_REVISION_183A) {
	4365	+ if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3465	4366	if (next == DWC3_LINK_STATE_U0) {
3466	4367	u32 u1u2;
3467	4368	u32 reg;
..	..	@@ -3480,7 +4381,7 @@
3480	4381
3481	4382	reg &= ~u1u2;
3482	4383
3483		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	4384	+ dwc3_gadget_dctl_write_safe(dwc, reg);
3484	4385	break;
3485	4386	default:
3486	4387	/* do nothing */
..	..	@@ -3514,8 +4415,7 @@
3514	4415	{
3515	4416	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3516	4417
3517		- if (dwc->link_state != next && next == DWC3_LINK_STATE_U3 &&
3518		- dwc->uwk_en)
	4418	+ if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
3519	4419	dwc3_suspend_gadget(dwc);
3520	4420
3521	4421	dwc->link_state = next;
..	..	@@ -3561,8 +4461,7 @@
3561	4461	dwc3_gadget_conndone_interrupt(dwc);
3562	4462	break;
3563	4463	case DWC3_DEVICE_EVENT_WAKEUP:
3564		- dev_dbg(dwc->dev, "device wakeup\n");
3565		- dwc3_gadget_wakeup_interrupt(dwc, event->event_info);
	4464	+ dwc3_gadget_wakeup_interrupt(dwc);
3566	4465	break;
3567	4466	case DWC3_DEVICE_EVENT_HIBER_REQ:
3568	4467	if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
..	..	@@ -3574,15 +4473,14 @@
3574	4473	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3575	4474	dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3576	4475	break;
3577		- case DWC3_DEVICE_EVENT_EOPF:
	4476	+ case DWC3_DEVICE_EVENT_SUSPEND:
3578	4477	/* It changed to be suspend event for version 2.30a and above */
3579		- if (dwc->revision >= DWC3_REVISION_230A) {
	4478	+ if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) {
3580	4479	/*
3581	4480	* Ignore suspend event until the gadget enters into
3582	4481	* USB_STATE_CONFIGURED state.
3583	4482	*/
3584		- dev_dbg(dwc->dev, "device suspend\n");
3585		- if (dwc->gadget.state >= USB_STATE_CONFIGURED)
	4483	+ if (dwc->gadget->state >= USB_STATE_CONFIGURED)
3586	4484	dwc3_gadget_suspend_interrupt(dwc,
3587	4485	event->event_info);
3588	4486	}
..	..	@@ -3615,7 +4513,6 @@
3615	4513	struct dwc3 *dwc = evt->dwc;
3616	4514	irqreturn_t ret = IRQ_NONE;
3617	4515	int left;
3618		- u32 reg;
3619	4516
3620	4517	left = evt->count;
3621	4518
..	..	@@ -3643,18 +4540,19 @@
3643	4540	}
3644	4541
3645	4542	evt->count = 0;
3646		- evt->flags &= ~DWC3_EVENT_PENDING;
3647	4543	ret = IRQ_HANDLED;
3648	4544
3649	4545	/* Unmask interrupt */
3650		- reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3651		- reg &= ~DWC3_GEVNTSIZ_INTMASK;
3652		- dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
	4546	+ dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
	4547	+ DWC3_GEVNTSIZ_SIZE(evt->length));
3653	4548
3654	4549	if (dwc->imod_interval) {
3655	4550	dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3656	4551	dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3657	4552	}
	4553	+
	4554	+ /* Keep the clearing of DWC3_EVENT_PENDING at the end */
	4555	+ evt->flags &= ~DWC3_EVENT_PENDING;
3658	4556
3659	4557	return ret;
3660	4558	}
..	..	@@ -3680,7 +4578,6 @@
3680	4578	struct dwc3 *dwc = evt->dwc;
3681	4579	u32 amount;
3682	4580	u32 count;
3683		- u32 reg;
3684	4581
3685	4582	if (pm_runtime_suspended(dwc->dev)) {
3686	4583	pm_runtime_get(dwc->dev);
..	..	@@ -3707,9 +4604,8 @@
3707	4604	evt->flags \|= DWC3_EVENT_PENDING;
3708	4605
3709	4606	/* Mask interrupt */
3710		- reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3711		- reg \|= DWC3_GEVNTSIZ_INTMASK;
3712		- dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
	4607	+ dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
	4608	+ DWC3_GEVNTSIZ_INTMASK \| DWC3_GEVNTSIZ_SIZE(evt->length));
3713	4609
3714	4610	amount = min(count, evt->length - evt->lpos);
3715	4611	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
..	..	@@ -3734,14 +4630,14 @@
3734	4630	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3735	4631	int irq;
3736	4632
3737		- irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
	4633	+ irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
3738	4634	if (irq > 0)
3739	4635	goto out;
3740	4636
3741	4637	if (irq == -EPROBE_DEFER)
3742	4638	goto out;
3743	4639
3744		- irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
	4640	+ irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
3745	4641	if (irq > 0)
3746	4642	goto out;
3747	4643
..	..	@@ -3752,14 +4648,18 @@
3752	4648	if (irq > 0)
3753	4649	goto out;
3754	4650
3755		- if (irq != -EPROBE_DEFER)
3756		- dev_err(dwc->dev, "missing peripheral IRQ\n");
3757		-
3758	4651	if (!irq)
3759	4652	irq = -EINVAL;
3760	4653
3761	4654	out:
3762	4655	return irq;
	4656	+}
	4657	+
	4658	+static void dwc_gadget_release(struct device *dev)
	4659	+{
	4660	+ struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
	4661	+
	4662	+ kfree(gadget);
3763	4663	}
3764	4664
3765	4665	/**
..	..	@@ -3772,6 +4672,7 @@
3772	4672	{
3773	4673	int ret;
3774	4674	int irq;
	4675	+ struct device *dev;
3775	4676
3776	4677	irq = dwc3_gadget_get_irq(dwc);
3777	4678	if (irq < 0) {
..	..	@@ -3804,17 +4705,22 @@
3804	4705	}
3805	4706
3806	4707	init_completion(&dwc->ep0_in_setup);
3807		- init_completion(&dwc->discon_done);
	4708	+ dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
	4709	+ if (!dwc->gadget) {
	4710	+ ret = -ENOMEM;
	4711	+ goto err3;
	4712	+ }
3808	4713
3809		- dwc->gadget.ops = &dwc3_gadget_ops;
3810		- dwc->gadget.speed = USB_SPEED_UNKNOWN;
3811		- dwc->gadget.sg_supported = true;
3812		- dwc->gadget.name = "dwc3-gadget";
3813		-#ifdef CONFIG_ARCH_ROCKCHIP
3814		- dwc->gadget.lpm_capable = false;
3815		-#else
3816		- dwc->gadget.lpm_capable = true;
3817		-#endif
	4714	+
	4715	+ usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
	4716	+ dev = &dwc->gadget->dev;
	4717	+ dev->platform_data = dwc;
	4718	+ dwc->gadget->ops = &dwc3_gadget_ops;
	4719	+ dwc->gadget->speed = USB_SPEED_UNKNOWN;
	4720	+ dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
	4721	+ dwc->gadget->sg_supported = true;
	4722	+ dwc->gadget->name = "dwc3-gadget";
	4723	+ dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable;
3818	4724
3819	4725	/*
3820	4726	* FIXME We might be setting max_speed to <SUPER, however versions
..	..	@@ -3832,12 +4738,13 @@
3832	4738	* is less than super speed because we don't have means, yet, to tell
3833	4739	* composite.c that we are USB 2.0 + LPM ECN.
3834	4740	*/
3835		- if (dwc->revision < DWC3_REVISION_220A &&
	4741	+ if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
3836	4742	!dwc->dis_metastability_quirk)
3837	4743	dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3838	4744	dwc->revision);
3839	4745
3840		- dwc->gadget.max_speed = dwc->maximum_speed;
	4746	+ dwc->gadget->max_speed = dwc->maximum_speed;
	4747	+ dwc->gadget->max_ssp_rate = dwc->max_ssp_rate;
3841	4748
3842	4749	/*
3843	4750	* REVISIT: Here we should clear all pending IRQs to be
..	..	@@ -3846,21 +4753,26 @@
3846	4753
3847	4754	ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3848	4755	if (ret)
3849		- goto err3;
3850		-
3851		- ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3852		- if (ret) {
3853		- dev_err(dwc->dev, "failed to register udc\n");
3854	4756	goto err4;
	4757	+
	4758	+ ret = usb_add_gadget(dwc->gadget);
	4759	+ if (ret) {
	4760	+ dev_err(dwc->dev, "failed to add gadget\n");
	4761	+ goto err5;
3855	4762	}
3856	4763
3857		- dwc3_gadget_set_speed(&dwc->gadget, dwc->maximum_speed);
	4764	+ if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
	4765	+ dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);
	4766	+ else
	4767	+ dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
3858	4768
3859	4769	return 0;
3860	4770
3861		-err4:
	4771	+err5:
3862	4772	dwc3_gadget_free_endpoints(dwc);
3863		-
	4773	+err4:
	4774	+ usb_put_gadget(dwc->gadget);
	4775	+ dwc->gadget = NULL;
3864	4776	err3:
3865	4777	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3866	4778	dwc->bounce_addr);
..	..	@@ -3880,8 +4792,12 @@
3880	4792
3881	4793	void dwc3_gadget_exit(struct dwc3 *dwc)
3882	4794	{
3883		- usb_del_gadget_udc(&dwc->gadget);
	4795	+ if (!dwc->gadget)
	4796	+ return;
	4797	+
	4798	+ usb_del_gadget(dwc->gadget);
3884	4799	dwc3_gadget_free_endpoints(dwc);
	4800	+ usb_put_gadget(dwc->gadget);
3885	4801	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3886	4802	dwc->bounce_addr);
3887	4803	kfree(dwc->setup_buf);
..	..	@@ -3891,21 +4807,27 @@
3891	4807
3892	4808	int dwc3_gadget_suspend(struct dwc3 *dwc)
3893	4809	{
	4810	+ unsigned long flags;
	4811	+
3894	4812	if (!dwc->gadget_driver)
3895	4813	return 0;
3896	4814
3897	4815	dwc3_gadget_run_stop(dwc, false, false);
	4816	+
	4817	+ spin_lock_irqsave(&dwc->lock, flags);
3898	4818	dwc3_disconnect_gadget(dwc);
3899	4819	__dwc3_gadget_stop(dwc);
	4820	+ spin_unlock_irqrestore(&dwc->lock, flags);
3900	4821
3901	4822	return 0;
3902	4823	}
3903	4824
3904	4825	int dwc3_gadget_resume(struct dwc3 *dwc)
3905	4826	{
	4827	+ struct dwc3_vendor *vdwc = container_of(dwc, struct dwc3_vendor, dwc);
3906	4828	int ret;
3907	4829
3908		- if (!dwc->gadget_driver)
	4830	+ if (!dwc->gadget_driver \|\| !vdwc->softconnect)
3909	4831	return 0;
3910	4832
3911	4833	ret = __dwc3_gadget_start(dwc);