~hc/RK356X_SDK_RELEASE.git

..	..	@@ -2,7 +2,7 @@
2	2	/*
3	3	* gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4	4	*
5		- * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
	5	+ * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6	6	*
7	7	* Authors: Felipe Balbi <balbi@ti.com>,
8	8	* Sebastian Andrzej Siewior <bigeasy@linutronix.de>
..	..	@@ -46,18 +46,18 @@
46	46	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47	47
48	48	switch (mode) {
49		- case TEST_J:
50		- case TEST_K:
51		- case TEST_SE0_NAK:
52		- case TEST_PACKET:
53		- case TEST_FORCE_EN:
	49	+ case USB_TEST_J:
	50	+ case USB_TEST_K:
	51	+ case USB_TEST_SE0_NAK:
	52	+ case USB_TEST_PACKET:
	53	+ case USB_TEST_FORCE_ENABLE:
54	54	reg \|= mode << 1;
55	55	break;
56	56	default:
57	57	return -EINVAL;
58	58	}
59	59
60		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	60	+ dwc3_gadget_dctl_write_safe(dwc, reg);
61	61
62	62	return 0;
63	63	}
..	..	@@ -95,7 +95,7 @@
95	95	* Wait until device controller is ready. Only applies to 1.94a and
96	96	* later RTL.
97	97	*/
98		- if (dwc->revision >= DWC3_REVISION_194A) {
	98	+ if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99	99	while (--retries) {
100	100	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101	101	if (reg & DWC3_DSTS_DCNRD)
..	..	@@ -111,6 +111,9 @@
111	111	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112	112	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113	113
	114	+ /* set no action before sending new link state change */
	115	+ dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	116	+
114	117	/* set requested state */
115	118	reg \|= DWC3_DCTL_ULSTCHNGREQ(state);
116	119	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
..	..	@@ -119,7 +122,7 @@
119	122	* The following code is racy when called from dwc3_gadget_wakeup,
120	123	* and is not needed, at least on newer versions
121	124	*/
122		- if (dwc->revision >= DWC3_REVISION_194A)
	125	+ if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
123	126	return 0;
124	127
125	128	/* wait for a change in DSTS */
..	..	@@ -135,182 +138,6 @@
135	138
136	139	return -ETIMEDOUT;
137	140	}
138		-
139		-#ifdef CONFIG_ARCH_ROCKCHIP
140		-/**
141		- * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
142		- * @dwc: pointer to our context structure
143		- *
144		- * This function will a best effort FIFO allocation in order
145		- * to improve FIFO usage and throughput, while still allowing
146		- * us to enable as many endpoints as possible.
147		- *
148		- * Keep in mind that this operation will be highly dependent
149		- * on the configured size for RAM1 - which contains TxFifo -,
150		- * the amount of endpoints enabled on coreConsultant tool, and
151		- * the width of the Master Bus.
152		- *
153		- * In the ideal world, we would always be able to satisfy the
154		- * following equation:
155		- *
156		- * ((512 + 2 * MDWIDTH-Bytes) + (Number of IN Endpoints - 1) * \
157		- * (3 * (1024 + MDWIDTH-Bytes) + MDWIDTH-Bytes)) / MDWIDTH-Bytes
158		- *
159		- * Unfortunately, due to many variables that's not always the case.
160		- */
161		-static int dwc3_gadget_resize_tx_fifos(struct dwc3 *dwc)
162		-{
163		- int last_fifo_depth;
164		- int fifo_size, total_size, total_resize = 0;
165		- int mdwidth;
166		- u8 num, fifo_number, num_in_eps;
167		-
168		- /*
169		- * Only support Tx fifos resize for gadget speed <= high speed
170		- * for the time being and do fifo resize operation only once
171		- * when connect done event occurs, because if resize Tx fifos
172		- * during controller transfer data, it may cause controller
173		- * run into abnormal and unrecoverable state.
174		- */
175		- if (!dwc->needs_fifo_resize \|\| dwc->fifo_resize_status)
176		- return 0;
177		-
178		- num_in_eps = DWC3_NUM_IN_EPS(&dwc->hwparams);
179		- mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
180		- /* MDWIDTH is represented in bits, we need it in bytes */
181		- mdwidth >>= 3;
182		- fifo_number = 0;
183		- /* Get the Tx FIFO 0 size and depth */
184		- fifo_size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
185		- last_fifo_depth = DWC3_GTXFIFOSIZ_TXFSTADDR(fifo_size) >> 16;
186		- /* Get the Tx FIFO (num_in_eps - 1) size and depth */
187		- fifo_size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num_in_eps - 1));
188		- /* Get the Tx FIFOs total size */
189		- total_size = (DWC3_GTXFIFOSIZ_TXFSTADDR(fifo_size) >> 16) +
190		- DWC3_GTXFIFOSIZ_TXFDEF(fifo_size) - last_fifo_depth;
191		-
192		- for (num = 0; num < dwc->num_eps; num++) {
193		- struct dwc3_ep *dep = dwc->eps[num];
194		- int mult = 1, maxpacket = 512;
195		- int tmp;
196		-
197		- /* Skip out endpoints */
198		- if (!dep \|\| !dep->direction)
199		- continue;
200		-
201		- switch (dep->endpoint.transfer_type) {
202		- case USB_ENDPOINT_XFER_CONTROL:
203		- if (!dep->endpoint.caps.type_control) {
204		- dev_dbg(dwc->dev, "%s may not be used\n",
205		- dep->name);
206		- goto out;
207		- }
208		-
209		- mult = 1;
210		- if (dwc->gadget.speed <= USB_SPEED_HIGH)
211		- maxpacket = 64;
212		- else
213		- maxpacket = 512;
214		- break;
215		- case USB_ENDPOINT_XFER_ISOC:
216		- if (!dep->endpoint.caps.type_iso) {
217		- dev_WARN(dwc->dev, "%s not support isoc type\n",
218		- dep->name);
219		- goto out;
220		- }
221		-
222		- /*
223		- * Set enough tx fifos for Isochronous endpoints
224		- * to get better performance and more compliance
225		- * with bus latency.
226		- */
227		- maxpacket = dep->endpoint.maxpacket;
228		- if (dwc->gadget.speed <= USB_SPEED_HIGH)
229		- mult = dep->endpoint.mult;
230		- else
231		- mult = dep->endpoint.mult *
232		- dep->endpoint.maxburst;
233		- mult = mult > 0 ? mult * 2 : 3;
234		- if (mult > 6)
235		- mult = 6;
236		- break;
237		- case USB_ENDPOINT_XFER_BULK:
238		- if (!dep->endpoint.caps.type_bulk) {
239		- dev_WARN(dwc->dev, "%s not support bulk type\n",
240		- dep->name);
241		- goto out;
242		- }
243		-
244		- /*
245		- * Set enough tx fifos for Bulk endpoints to get
246		- * better transmission performance.
247		- */
248		- mult = 3;
249		- if (dwc->gadget.speed <= USB_SPEED_HIGH) {
250		- maxpacket = 512;
251		- } else {
252		- if (dep->endpoint.maxburst > mult) {
253		- mult = dep->endpoint.maxburst;
254		- if (mult > 6)
255		- mult = 6;
256		- }
257		- maxpacket = 1024;
258		- }
259		- break;
260		- case USB_ENDPOINT_XFER_INT:
261		- /* Bulk endpoints handle interrupt transfers. */
262		- if (!dep->endpoint.caps.type_int &&
263		- !dep->endpoint.caps.type_bulk) {
264		- dev_WARN(dwc->dev, "%s not support int type\n",
265		- dep->name);
266		- goto out;
267		- }
268		-
269		- /*
270		- * REVIST: we assume that the maxpacket of interrupt
271		- * endpoint is 64 Bytes for MTP and the other functions.
272		- */
273		- mult = 1;
274		- maxpacket = 64;
275		- break;
276		- default:
277		- /*
278		- * This is only possible with faulty memory
279		- * because we checked it already.
280		- */
281		- dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
282		- dep->endpoint.transfer_type);
283		- goto out;
284		- }
285		-
286		- tmp = mult * (maxpacket + mdwidth);
287		- tmp += mdwidth;
288		-
289		- fifo_size = DIV_ROUND_UP(tmp, mdwidth);
290		- total_resize += fifo_size;
291		- fifo_size \|= (last_fifo_depth << 16);
292		-
293		- if (total_resize > total_size) {
294		- dev_WARN(dwc->dev, "Tx FIFO resize overflow!\n");
295		- break;
296		- }
297		-
298		- dev_dbg(dwc->dev, "%s: FIFO Addr %04x Size %d\n",
299		- dep->name, last_fifo_depth, fifo_size & 0xffff);
300		-
301		- dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(fifo_number),
302		- fifo_size);
303		-
304		- last_fifo_depth += (fifo_size & 0xffff);
305		- fifo_number++;
306		- }
307		-
308		-out:
309		- dwc->fifo_resize_status = true;
310		-
311		- return 0;
312		-}
313		-#endif
314	141
315	142	/**
316	143	* dwc3_ep_inc_trb - increment a trb index.
..	..	@@ -353,6 +180,7 @@
353	180	list_del(&req->list);
354	181	req->remaining = 0;
355	182	req->needs_extra_trb = false;
	183	+ req->num_trbs = 0;
356	184
357	185	if (req->request.status == -EINPROGRESS)
358	186	req->request.status = status;
..	..	@@ -400,7 +228,8 @@
400	228	* Caller should take care of locking. Issue @cmd with a given @param to @dwc
401	229	* and wait for its completion.
402	230	*/
403		-int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
	231	+int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
	232	+ u32 param)
404	233	{
405	234	u32 timeout = 500;
406	235	int status = 0;
..	..	@@ -441,7 +270,7 @@
441	270	* Caller should handle locking. This function will issue @cmd with given
442	271	* @params to @dep and wait for its completion.
443	272	*/
444		-int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
	273	+int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
445	274	struct dwc3_gadget_ep_cmd_params *params)
446	275	{
447	276	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
..	..	@@ -463,7 +292,8 @@
463	292	*
464	293	* DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
465	294	*/
466		- if (dwc->gadget.speed <= USB_SPEED_HIGH) {
	295	+ if (dwc->gadget->speed <= USB_SPEED_HIGH \|\|
	296	+ DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) {
467	297	reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
468	298	if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
469	299	saved_config \|= DWC3_GUSB2PHYCFG_SUSPHY;
..	..	@@ -482,19 +312,38 @@
482	312	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
483	313	int link_state;
484	314
	315	+ /*
	316	+ * Initiate remote wakeup if the link state is in U3 when
	317	+ * operating in SS/SSP or L1/L2 when operating in HS/FS. If the
	318	+ * link state is in U1/U2, no remote wakeup is needed. The Start
	319	+ * Transfer command will initiate the link recovery.
	320	+ */
485	321	link_state = dwc3_gadget_get_link_state(dwc);
486		- if (link_state == DWC3_LINK_STATE_U1 \|\|
487		- link_state == DWC3_LINK_STATE_U2 \|\|
488		- link_state == DWC3_LINK_STATE_U3) {
	322	+ switch (link_state) {
	323	+ case DWC3_LINK_STATE_U2:
	324	+ if (dwc->gadget->speed >= USB_SPEED_SUPER)
	325	+ break;
	326	+
	327	+ fallthrough;
	328	+ case DWC3_LINK_STATE_U3:
489	329	ret = __dwc3_gadget_wakeup(dwc);
490	330	dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
491	331	ret);
	332	+ break;
492	333	}
493	334	}
494	335
495		- dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
496		- dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
497		- dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
	336	+ /*
	337	+ * For some commands such as Update Transfer command, DEPCMDPARn
	338	+ * registers are reserved. Since the driver often sends Update Transfer
	339	+ * command, don't write to DEPCMDPARn to avoid register write delays and
	340	+ * improve performance.
	341	+ */
	342	+ if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
	343	+ dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
	344	+ dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
	345	+ dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
	346	+ }
498	347
499	348	/*
500	349	* Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
..	..	@@ -518,6 +367,14 @@
518	367	cmd \|= DWC3_DEPCMD_CMDACT;
519	368
520	369	dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
	370	+
	371	+ if (!(cmd & DWC3_DEPCMD_CMDACT) \|\|
	372	+ (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
	373	+ !(cmd & DWC3_DEPCMD_CMDIOC))) {
	374	+ ret = 0;
	375	+ goto skip_status;
	376	+ }
	377	+
521	378	do {
522	379	reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
523	380	if (!(reg & DWC3_DEPCMD_CMDACT)) {
..	..	@@ -528,6 +385,8 @@
528	385	ret = 0;
529	386	break;
530	387	case DEPEVT_TRANSFER_NO_RESOURCE:
	388	+ dev_WARN(dwc->dev, "No resource for %s\n",
	389	+ dep->name);
531	390	ret = -EINVAL;
532	391	break;
533	392	case DEPEVT_TRANSFER_BUS_EXPIRY:
..	..	@@ -557,11 +416,15 @@
557	416	cmd_status = -ETIMEDOUT;
558	417	}
559	418
	419	+skip_status:
560	420	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
561	421
562		- if (ret == 0 && DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
563		- dep->flags \|= DWC3_EP_TRANSFER_STARTED;
564		- dwc3_gadget_ep_get_transfer_index(dep);
	422	+ if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
	423	+ if (ret == 0)
	424	+ dep->flags \|= DWC3_EP_TRANSFER_STARTED;
	425	+
	426	+ if (ret != -ETIMEDOUT)
	427	+ dwc3_gadget_ep_get_transfer_index(dep);
565	428	}
566	429
567	430	if (saved_config) {
..	..	@@ -572,6 +435,7 @@
572	435
573	436	return ret;
574	437	}
	438	+EXPORT_SYMBOL_GPL(dwc3_send_gadget_ep_cmd);
575	439
576	440	static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
577	441	{
..	..	@@ -587,8 +451,9 @@
587	451	* IN transfers due to a mishandled error condition. Synopsys
588	452	* STAR 9000614252.
589	453	*/
590		- if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
591		- (dwc->gadget.speed >= USB_SPEED_SUPER))
	454	+ if (dep->direction &&
	455	+ !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
	456	+ (dwc->gadget->speed >= USB_SPEED_SUPER))
592	457	cmd \|= DWC3_DEPCMD_CLEARPENDIN;
593	458
594	459	memset(&params, 0, sizeof(params));
..	..	@@ -728,8 +593,9 @@
728	593	\| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
729	594
730	595	/* Burst size is only needed in SuperSpeed mode */
731		- if (dwc->gadget.speed >= USB_SPEED_SUPER) {
	596	+ if (dwc->gadget->speed >= USB_SPEED_SUPER) {
732	597	u32 burst = dep->endpoint.maxburst;
	598	+
733	599	params.param0 \|= DWC3_DEPCFG_BURST_SIZE(burst - 1);
734	600	}
735	601
..	..	@@ -745,6 +611,7 @@
745	611
746	612	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
747	613	params.param1 \|= DWC3_DEPCFG_STREAM_CAPABLE
	614	+ \| DWC3_DEPCFG_XFER_COMPLETE_EN
748	615	\| DWC3_DEPCFG_STREAM_EVENT_EN;
749	616	dep->stream_capable = true;
750	617	}
..	..	@@ -781,7 +648,7 @@
781	648	bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
782	649
783	650	if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
784		- dwc->gadget.speed == USB_SPEED_FULL)
	651	+ dwc->gadget->speed == USB_SPEED_FULL)
785	652	dep->interval = desc->bInterval;
786	653	else
787	654	dep->interval = 1 << (desc->bInterval - 1);
..	..	@@ -790,6 +657,335 @@
790	657	}
791	658
792	659	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
	660	+}
	661	+
	662	+/**
	663	+ * dwc3_gadget_get_tx_fifos_size - Get the txfifos total size
	664	+ * @dwc: pointer to the DWC3 context
	665	+ *
	666	+ * 3-RAM configuration:
	667	+ * RAM0 depth = Descriptor Cache depth
	668	+ * RAM1 depth = TxFIFOs depth
	669	+ * RAM2 depth = RxFIFOs depth
	670	+ *
	671	+ * 2-RAM configuration:
	672	+ * RAM0 depth = Descriptor Cache depth + RxFIFOs depth
	673	+ * RAM1 depth = TxFIFOs depth
	674	+ *
	675	+ * 1-RAM configuration:
	676	+ * RAM0 depth = Descriptor Cache depth + RxFIFOs depth + TxFIFOs depth
	677	+ */
	678	+static int dwc3_gadget_get_tx_fifos_size(struct dwc3 *dwc)
	679	+{
	680	+ int txfifo_depth = 0;
	681	+ int ram0_depth, rxfifo_size;
	682	+
	683	+ /* Get the depth of the TxFIFOs */
	684	+ if (DWC3_NUM_RAMS(dwc->hwparams.hwparams1) > 1) {
	685	+ /* For 2 or 3-RAM, RAM1 contains TxFIFOs */
	686	+ txfifo_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
	687	+ } else {
	688	+ /* For 1-RAM, RAM0 contains Descriptor Cache, RxFIFOs, and TxFIFOs */
	689	+ ram0_depth = DWC3_GHWPARAMS6_RAM0_DEPTH(dwc->hwparams.hwparams6);
	690	+
	691	+ /* All OUT endpoints share a single RxFIFO space */
	692	+ rxfifo_size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
	693	+ if (DWC3_IP_IS(DWC3))
	694	+ txfifo_depth = ram0_depth - DWC3_GRXFIFOSIZ_RXFDEP(rxfifo_size);
	695	+ else
	696	+ txfifo_depth = ram0_depth - DWC31_GRXFIFOSIZ_RXFDEP(rxfifo_size);
	697	+
	698	+ /* The value of GRxFIFOSIZ0[31:16] is the depth of Descriptor Cache */
	699	+ txfifo_depth -= DWC3_GRXFIFOSIZ_RXFSTADDR(rxfifo_size) >> 16;
	700	+ }
	701	+
	702	+ return txfifo_depth;
	703	+}
	704	+
	705	+/**
	706	+ * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
	707	+ * @dwc: pointer to the DWC3 context
	708	+ * @nfifos: number of fifos to calculate for
	709	+ *
	710	+ * Calculates the size value based on the equation below:
	711	+ *
	712	+ * DWC3 revision 280A and prior:
	713	+ * fifo_size = mult * (max_packet / mdwidth) + 1;
	714	+ *
	715	+ * DWC3 revision 290A and onwards:
	716	+ * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
	717	+ *
	718	+ * The max packet size is set to 1024, as the txfifo requirements mainly apply
	719	+ * to super speed USB use cases. However, it is safe to overestimate the fifo
	720	+ * allocations for other scenarios, i.e. high speed USB.
	721	+ */
	722	+static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
	723	+{
	724	+ int max_packet = 1024;
	725	+ int fifo_size;
	726	+ int mdwidth;
	727	+
	728	+ mdwidth = dwc3_mdwidth(dwc);
	729	+
	730	+ /* MDWIDTH is represented in bits, we need it in bytes */
	731	+ mdwidth >>= 3;
	732	+
	733	+ if (DWC3_VER_IS_PRIOR(DWC3, 290A))
	734	+ fifo_size = mult * (max_packet / mdwidth) + 1;
	735	+ else
	736	+ fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
	737	+ return fifo_size;
	738	+}
	739	+
	740	+/**
	741	+ * dwc3_gadget_clear_tx_fifo_size - Clears txfifo allocation
	742	+ * @dwc: pointer to the DWC3 context
	743	+ *
	744	+ * Iterates through all the endpoint registers and clears the previous txfifo
	745	+ * allocations.
	746	+ */
	747	+void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
	748	+{
	749	+ struct dwc3_ep *dep;
	750	+ int fifo_depth;
	751	+ int size;
	752	+ int num;
	753	+
	754	+ if (!dwc->do_fifo_resize)
	755	+ return;
	756	+
	757	+ /* Read ep0IN related TXFIFO size */
	758	+ dep = dwc->eps[1];
	759	+ size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
	760	+ if (DWC3_IP_IS(DWC3))
	761	+ fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
	762	+ else
	763	+ fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
	764	+
	765	+ dwc->last_fifo_depth = fifo_depth;
	766	+ /* Clear existing TXFIFO for all IN eps except ep0 */
	767	+ for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
	768	+ num += 2) {
	769	+ dep = dwc->eps[num];
	770	+ /* Don't change TXFRAMNUM on usb31 version */
	771	+ size = DWC3_IP_IS(DWC3) ? 0 :
	772	+ dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
	773	+ DWC31_GTXFIFOSIZ_TXFRAMNUM;
	774	+
	775	+ dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
	776	+ dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
	777	+ }
	778	+ dwc->num_ep_resized = 0;
	779	+}
	780	+
	781	+/**
	782	+ * __dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for Rockchip platform
	783	+ *
	784	+ * @dep: pointer to dwc3_ep structure
	785	+ *
	786	+ * According to the different USB transfer type and Speed,
	787	+ * this function will a best effort FIFO allocation in order
	788	+ * to improve FIFO usage and throughput, while still allowing
	789	+ * us to enable as many endpoints as possible.
	790	+ */
	791	+static int __dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
	792	+{
	793	+ struct dwc3 *dwc = dep->dwc;
	794	+ u32 fifo_0_start, last_fifo_depth, ram1_depth;
	795	+ u32 fifo_size, maxpacket, mdwidth, mult;
	796	+ u32 tmp;
	797	+
	798	+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
	799	+ /*
	800	+ * Set enough tx fifos for Isochronous endpoints to get better
	801	+ * performance and more compliance with bus latency.
	802	+ */
	803	+ maxpacket = dep->endpoint.maxpacket;
	804	+ if (gadget_is_superspeed(dwc->gadget))
	805	+ mult = dep->endpoint.mult * dep->endpoint.maxburst;
	806	+ else
	807	+ mult = dep->endpoint.mult;
	808	+
	809	+ mult = mult > 0 ? mult * 2 : 3;
	810	+ if (mult > 6)
	811	+ mult = 6;
	812	+ } else if (usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
	813	+ /*
	814	+ * Set enough tx fifos for Bulk endpoints to get
	815	+ * better transmission performance.
	816	+ */
	817	+ mult = 3;
	818	+ if (gadget_is_superspeed(dwc->gadget)) {
	819	+ if (dep->endpoint.maxburst > mult) {
	820	+ mult = dep->endpoint.maxburst;
	821	+ if (mult > 6)
	822	+ mult = 6;
	823	+ }
	824	+ maxpacket = 1024;
	825	+ } else {
	826	+ maxpacket = 512;
	827	+ }
	828	+ } else if (usb_endpoint_xfer_int(dep->endpoint.desc)) {
	829	+ /*
	830	+ * REVIST: we assume that the maxpacket of interrupt
	831	+ * endpoint is 64 Bytes for MTP and the other functions.
	832	+ */
	833	+ mult = 1;
	834	+ maxpacket = 64;
	835	+ } else {
	836	+ goto out;
	837	+ }
	838	+
	839	+ mdwidth = dwc3_mdwidth(dwc);
	840	+ mdwidth >>= 3; /* bits convert to bytes */
	841	+ ram1_depth = dwc3_gadget_get_tx_fifos_size(dwc);
	842	+ last_fifo_depth = dwc->last_fifo_depth;
	843	+
	844	+ /* Calculate the fifo size for this EP */
	845	+ tmp = mult * (maxpacket + mdwidth);
	846	+ tmp += mdwidth;
	847	+ fifo_size = DIV_ROUND_UP(tmp, mdwidth);
	848	+
	849	+ /* Check if TXFIFOs start at non-zero addr */
	850	+ tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
	851	+ fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
	852	+ fifo_size \|= (fifo_0_start + (last_fifo_depth << 16));
	853	+
	854	+ if (DWC3_IP_IS(DWC3))
	855	+ last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
	856	+ else
	857	+ last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
	858	+
	859	+ /* Check fifo size allocation doesn't exceed available RAM size. */
	860	+ if (last_fifo_depth >= ram1_depth) {
	861	+ dev_err(dwc->dev, "Fifosize(0x%x) > RAM size(0x%x) %s depth(0x%x)\n",
	862	+ last_fifo_depth, ram1_depth,
	863	+ dep->endpoint.name, fifo_size & 0xfff);
	864	+ return -ENOMEM;
	865	+ }
	866	+
	867	+ dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
	868	+ dep->flags \|= DWC3_EP_TXFIFO_RESIZED;
	869	+ dwc->last_fifo_depth = last_fifo_depth;
	870	+ dwc->num_ep_resized++;
	871	+
	872	+out:
	873	+ return 0;
	874	+}
	875	+
	876	+/*
	877	+ * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
	878	+ * @dwc: pointer to our context structure
	879	+ *
	880	+ * This function will a best effort FIFO allocation in order
	881	+ * to improve FIFO usage and throughput, while still allowing
	882	+ * us to enable as many endpoints as possible.
	883	+ *
	884	+ * Keep in mind that this operation will be highly dependent
	885	+ * on the configured size for RAM1 - which contains TxFifo -,
	886	+ * the amount of endpoints enabled on coreConsultant tool, and
	887	+ * the width of the Master Bus.
	888	+ *
	889	+ * In general, FIFO depths are represented with the following equation:
	890	+ *
	891	+ * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
	892	+ *
	893	+ * In conjunction with dwc3_gadget_check_config(), this resizing logic will
	894	+ * ensure that all endpoints will have enough internal memory for one max
	895	+ * packet per endpoint.
	896	+ */
	897	+static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
	898	+{
	899	+ struct dwc3 *dwc = dep->dwc;
	900	+ int fifo_0_start;
	901	+ int ram1_depth;
	902	+ int fifo_size;
	903	+ int min_depth;
	904	+ int num_in_ep;
	905	+ int remaining;
	906	+ int num_fifos = 1;
	907	+ int fifo;
	908	+ int tmp;
	909	+
	910	+ if (!dwc->do_fifo_resize)
	911	+ return 0;
	912	+
	913	+ /* resize IN endpoints except ep0 */
	914	+ if (!usb_endpoint_dir_in(dep->endpoint.desc) \|\| dep->number <= 1)
	915	+ return 0;
	916	+
	917	+ /* bail if already resized */
	918	+ if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
	919	+ return 0;
	920	+
	921	+ if (IS_REACHABLE(CONFIG_ARCH_ROCKCHIP))
	922	+ return __dwc3_gadget_resize_tx_fifos(dep);
	923	+
	924	+ ram1_depth = dwc3_gadget_get_tx_fifos_size(dwc);
	925	+
	926	+ if ((dep->endpoint.maxburst > 1 &&
	927	+ usb_endpoint_xfer_bulk(dep->endpoint.desc)) \|\|
	928	+ usb_endpoint_xfer_isoc(dep->endpoint.desc))
	929	+ num_fifos = 3;
	930	+
	931	+ if (dep->endpoint.maxburst > 6 &&
	932	+ (usb_endpoint_xfer_bulk(dep->endpoint.desc) \|\|
	933	+ usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
	934	+ num_fifos = dwc->tx_fifo_resize_max_num;
	935	+
	936	+ /* FIFO size for a single buffer */
	937	+ fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1);
	938	+
	939	+ /* Calculate the number of remaining EPs w/o any FIFO */
	940	+ num_in_ep = dwc->max_cfg_eps;
	941	+ num_in_ep -= dwc->num_ep_resized;
	942	+
	943	+ /* Reserve at least one FIFO for the number of IN EPs */
	944	+ min_depth = num_in_ep * (fifo + 1);
	945	+ remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
	946	+ remaining = max_t(int, 0, remaining);
	947	+ /*
	948	+ * We've already reserved 1 FIFO per EP, so check what we can fit in
	949	+ * addition to it. If there is not enough remaining space, allocate
	950	+ * all the remaining space to the EP.
	951	+ */
	952	+ fifo_size = (num_fifos - 1) * fifo;
	953	+ if (remaining < fifo_size)
	954	+ fifo_size = remaining;
	955	+
	956	+ fifo_size += fifo;
	957	+ /* Last increment according to the TX FIFO size equation */
	958	+ fifo_size++;
	959	+
	960	+ /* Check if TXFIFOs start at non-zero addr */
	961	+ tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
	962	+ fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
	963	+
	964	+ fifo_size \|= (fifo_0_start + (dwc->last_fifo_depth << 16));
	965	+ if (DWC3_IP_IS(DWC3))
	966	+ dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
	967	+ else
	968	+ dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
	969	+
	970	+ /* Check fifo size allocation doesn't exceed available RAM size. */
	971	+ if (dwc->last_fifo_depth >= ram1_depth) {
	972	+ dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
	973	+ dwc->last_fifo_depth, ram1_depth,
	974	+ dep->endpoint.name, fifo_size);
	975	+ if (DWC3_IP_IS(DWC3))
	976	+ fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
	977	+ else
	978	+ fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
	979	+
	980	+ dwc->last_fifo_depth -= fifo_size;
	981	+ return -ENOMEM;
	982	+ }
	983	+
	984	+ dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
	985	+ dep->flags \|= DWC3_EP_TXFIFO_RESIZED;
	986	+ dwc->num_ep_resized++;
	987	+
	988	+ return 0;
793	989	}
794	990
795	991	/**
..	..	@@ -809,6 +1005,10 @@
809	1005	int ret;
810	1006
811	1007	if (!(dep->flags & DWC3_EP_ENABLED)) {
	1008	+ ret = dwc3_gadget_resize_tx_fifos(dep);
	1009	+ if (ret)
	1010	+ return ret;
	1011	+
812	1012	ret = dwc3_gadget_start_config(dep);
813	1013	if (ret)
814	1014	return ret;
..	..	@@ -829,12 +1029,13 @@
829	1029	reg \|= DWC3_DALEPENA_EP(dep->number);
830	1030	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
831	1031
	1032	+ dep->trb_dequeue = 0;
	1033	+ dep->trb_enqueue = 0;
	1034	+
832	1035	if (usb_endpoint_xfer_control(desc))
833	1036	goto out;
834	1037
835	1038	/* Initialize the TRB ring */
836		- dep->trb_dequeue = 0;
837		- dep->trb_enqueue = 0;
838	1039	memset(dep->trb_pool, 0,
839	1040	sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
840	1041
..	..	@@ -852,7 +1053,7 @@
852	1053	* Issue StartTransfer here with no-op TRB so we can always rely on No
853	1054	* Response Update Transfer command.
854	1055	*/
855		- if ((usb_endpoint_xfer_bulk(desc) && !dep->stream_capable) \|\|
	1056	+ if (usb_endpoint_xfer_bulk(desc) \|\|
856	1057	usb_endpoint_xfer_int(desc)) {
857	1058	struct dwc3_gadget_ep_cmd_params params;
858	1059	struct dwc3_trb *trb;
..	..	@@ -871,6 +1072,37 @@
871	1072	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
872	1073	if (ret < 0)
873	1074	return ret;
	1075	+
	1076	+ if (dep->stream_capable) {
	1077	+ /*
	1078	+ * For streams, at start, there maybe a race where the
	1079	+ * host primes the endpoint before the function driver
	1080	+ * queues a request to initiate a stream. In that case,
	1081	+ * the controller will not see the prime to generate the
	1082	+ * ERDY and start stream. To workaround this, issue a
	1083	+ * no-op TRB as normal, but end it immediately. As a
	1084	+ * result, when the function driver queues the request,
	1085	+ * the next START_TRANSFER command will cause the
	1086	+ * controller to generate an ERDY to initiate the
	1087	+ * stream.
	1088	+ */
	1089	+ dwc3_stop_active_transfer(dep, true, true);
	1090	+
	1091	+ /*
	1092	+ * All stream eps will reinitiate stream on NoStream
	1093	+ * rejection until we can determine that the host can
	1094	+ * prime after the first transfer.
	1095	+ *
	1096	+ * However, if the controller is capable of
	1097	+ * TXF_FLUSH_BYPASS, then IN direction endpoints will
	1098	+ * automatically restart the stream without the driver
	1099	+ * initiation.
	1100	+ */
	1101	+ if (!dep->direction \|\|
	1102	+ !(dwc->hwparams.hwparams9 &
	1103	+ DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
	1104	+ dep->flags \|= DWC3_EP_FORCE_RESTART_STREAM;
	1105	+ }
874	1106	}
875	1107
876	1108	out:
..	..	@@ -879,31 +1111,33 @@
879	1111	return 0;
880	1112	}
881	1113
882		-static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
883		- bool interrupt);
884		-static void dwc3_remove_requests(struct dwc3 dwc, struct dwc3_ep dep)
	1114	+void dwc3_remove_requests(struct dwc3 dwc, struct dwc3_ep dep, int status)
885	1115	{
886	1116	struct dwc3_request *req;
887	1117
888	1118	dwc3_stop_active_transfer(dep, true, false);
889	1119
	1120	+ /* If endxfer is delayed, avoid unmapping requests */
	1121	+ if (dep->flags & DWC3_EP_DELAY_STOP)
	1122	+ return;
	1123	+
890	1124	/* - giveback all requests to gadget driver */
891	1125	while (!list_empty(&dep->started_list)) {
892	1126	req = next_request(&dep->started_list);
893	1127
894		- dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
	1128	+ dwc3_gadget_giveback(dep, req, status);
895	1129	}
896	1130
897	1131	while (!list_empty(&dep->pending_list)) {
898	1132	req = next_request(&dep->pending_list);
899	1133
900		- dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
	1134	+ dwc3_gadget_giveback(dep, req, status);
901	1135	}
902	1136
903	1137	while (!list_empty(&dep->cancelled_list)) {
904	1138	req = next_request(&dep->cancelled_list);
905	1139
906		- dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
	1140	+ dwc3_gadget_giveback(dep, req, status);
907	1141	}
908	1142	}
909	1143
..	..	@@ -921,10 +1155,9 @@
921	1155	{
922	1156	struct dwc3 *dwc = dep->dwc;
923	1157	u32 reg;
	1158	+ u32 mask;
924	1159
925	1160	trace_dwc3_gadget_ep_disable(dep);
926		-
927		- dwc3_remove_requests(dwc, dep);
928	1161
929	1162	/* make sure HW endpoint isn't stalled */
930	1163	if (dep->flags & DWC3_EP_STALL)
..	..	@@ -934,9 +1167,19 @@
934	1167	reg &= ~DWC3_DALEPENA_EP(dep->number);
935	1168	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
936	1169
	1170	+ dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
	1171	+
937	1172	dep->stream_capable = false;
938	1173	dep->type = 0;
939		- dep->flags = 0;
	1174	+ mask = DWC3_EP_TXFIFO_RESIZED;
	1175	+ /*
	1176	+ * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is
	1177	+ * set. Do not clear DEP flags, so that the end transfer command will
	1178	+ * be reattempted during the next SETUP stage.
	1179	+ */
	1180	+ if (dep->flags & DWC3_EP_DELAY_STOP)
	1181	+ mask \|= (DWC3_EP_DELAY_STOP \| DWC3_EP_TRANSFER_STARTED);
	1182	+ dep->flags &= mask;
940	1183
941	1184	/* Clear out the ep descriptors for non-ep0 */
942	1185	if (dep->number > 1) {
..	..	@@ -1099,15 +1342,49 @@
1099	1342	return trbs_left;
1100	1343	}
1101	1344
1102		-static void __dwc3_prepare_one_trb(struct dwc3_ep dep, struct dwc3_trb trb,
1103		- dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
1104		- unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
	1345	+/**
	1346	+ * dwc3_prepare_one_trb - setup one TRB from one request
	1347	+ * @dep: endpoint for which this request is prepared
	1348	+ * @req: dwc3_request pointer
	1349	+ * @trb_length: buffer size of the TRB
	1350	+ * @chain: should this TRB be chained to the next?
	1351	+ * @node: only for isochronous endpoints. First TRB needs different type.
	1352	+ * @use_bounce_buffer: set to use bounce buffer
	1353	+ * @must_interrupt: set to interrupt on TRB completion
	1354	+ */
	1355	+static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
	1356	+ struct dwc3_request *req, unsigned int trb_length,
	1357	+ unsigned int chain, unsigned int node, bool use_bounce_buffer,
	1358	+ bool must_interrupt)
1105	1359	{
	1360	+ struct dwc3_trb *trb;
	1361	+ dma_addr_t dma;
	1362	+ unsigned int stream_id = req->request.stream_id;
	1363	+ unsigned int short_not_ok = req->request.short_not_ok;
	1364	+ unsigned int no_interrupt = req->request.no_interrupt;
	1365	+ unsigned int is_last = req->request.is_last;
1106	1366	struct dwc3 *dwc = dep->dwc;
1107		- struct usb_gadget *gadget = &dwc->gadget;
	1367	+ struct usb_gadget *gadget = dwc->gadget;
1108	1368	enum usb_device_speed speed = gadget->speed;
1109	1369
1110		- trb->size = DWC3_TRB_SIZE_LENGTH(length);
	1370	+ if (use_bounce_buffer)
	1371	+ dma = dep->dwc->bounce_addr;
	1372	+ else if (req->request.num_sgs > 0)
	1373	+ dma = sg_dma_address(req->start_sg);
	1374	+ else
	1375	+ dma = req->request.dma;
	1376	+
	1377	+ trb = &dep->trb_pool[dep->trb_enqueue];
	1378	+
	1379	+ if (!req->trb) {
	1380	+ dwc3_gadget_move_started_request(req);
	1381	+ req->trb = trb;
	1382	+ req->trb_dma = dwc3_trb_dma_offset(dep, trb);
	1383	+ }
	1384	+
	1385	+ req->num_trbs++;
	1386	+
	1387	+ trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1111	1388	trb->bpl = lower_32_bits(dma);
1112	1389	trb->bph = upper_32_bits(dma);
1113	1390
..	..	@@ -1147,10 +1424,10 @@
1147	1424	unsigned int mult = 2;
1148	1425	unsigned int maxp = usb_endpoint_maxp(ep->desc);
1149	1426
1150		- if (length <= (2 * maxp))
	1427	+ if (req->request.length <= (2 * maxp))
1151	1428	mult--;
1152	1429
1153		- if (length <= maxp)
	1430	+ if (req->request.length <= maxp)
1154	1431	mult--;
1155	1432
1156	1433	trb->size \|= DWC3_TRB_SIZE_PCM1(mult);
..	..	@@ -1159,8 +1436,8 @@
1159	1436	trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1160	1437	}
1161	1438
1162		- /* always enable Interrupt on Missed ISOC */
1163		- trb->ctrl \|= DWC3_TRB_CTRL_ISP_IMI;
	1439	+ if (!no_interrupt && !chain)
	1440	+ trb->ctrl \|= DWC3_TRB_CTRL_ISP_IMI;
1164	1441	break;
1165	1442
1166	1443	case USB_ENDPOINT_XFER_BULK:
..	..	@@ -1188,12 +1465,13 @@
1188	1465	trb->ctrl \|= DWC3_TRB_CTRL_ISP_IMI;
1189	1466	}
1190	1467
1191		- if ((!no_interrupt && !chain) \|\|
1192		- (dwc3_calc_trbs_left(dep) == 1))
	1468	+ if ((!no_interrupt && !chain) \|\| must_interrupt)
1193	1469	trb->ctrl \|= DWC3_TRB_CTRL_IOC;
1194	1470
1195	1471	if (chain)
1196	1472	trb->ctrl \|= DWC3_TRB_CTRL_CHN;
	1473	+ else if (dep->stream_capable && is_last)
	1474	+ trb->ctrl \|= DWC3_TRB_CTRL_LST;
1197	1475
1198	1476	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1199	1477	trb->ctrl \|= DWC3_TRB_CTRL_SID_SOFN(stream_id);
..	..	@@ -1218,54 +1496,69 @@
1218	1496	trace_dwc3_prepare_trb(dep, trb);
1219	1497	}
1220	1498
1221		-/**
1222		- * dwc3_prepare_one_trb - setup one TRB from one request
1223		- * @dep: endpoint for which this request is prepared
1224		- * @req: dwc3_request pointer
1225		- * @trb_length: buffer size of the TRB
1226		- * @chain: should this TRB be chained to the next?
1227		- * @node: only for isochronous endpoints. First TRB needs different type.
1228		- */
1229		-static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1230		- struct dwc3_request *req, unsigned int trb_length,
1231		- unsigned chain, unsigned node)
	1499	+static bool dwc3_needs_extra_trb(struct dwc3_ep dep, struct dwc3_request req)
1232	1500	{
1233		- struct dwc3_trb *trb;
1234		- dma_addr_t dma;
1235		- unsigned stream_id = req->request.stream_id;
1236		- unsigned short_not_ok = req->request.short_not_ok;
1237		- unsigned no_interrupt = req->request.no_interrupt;
	1501	+ unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
	1502	+ unsigned int rem = req->request.length % maxp;
1238	1503
1239		- if (req->request.num_sgs > 0)
1240		- dma = sg_dma_address(req->start_sg);
1241		- else
1242		- dma = req->request.dma;
	1504	+ if ((req->request.length && req->request.zero && !rem &&
	1505	+ !usb_endpoint_xfer_isoc(dep->endpoint.desc)) \|\|
	1506	+ (!req->direction && rem))
	1507	+ return true;
1243	1508
1244		- trb = &dep->trb_pool[dep->trb_enqueue];
1245		-
1246		- if (!req->trb) {
1247		- dwc3_gadget_move_started_request(req);
1248		- req->trb = trb;
1249		- req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1250		- }
1251		-
1252		- req->num_trbs++;
1253		-
1254		- __dwc3_prepare_one_trb(dep, trb, dma, trb_length, chain, node,
1255		- stream_id, short_not_ok, no_interrupt);
	1509	+ return false;
1256	1510	}
1257	1511
1258		-static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
	1512	+/**
	1513	+ * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
	1514	+ * @dep: The endpoint that the request belongs to
	1515	+ * @req: The request to prepare
	1516	+ * @entry_length: The last SG entry size
	1517	+ * @node: Indicates whether this is not the first entry (for isoc only)
	1518	+ *
	1519	+ * Return the number of TRBs prepared.
	1520	+ */
	1521	+static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
	1522	+ struct dwc3_request *req, unsigned int entry_length,
	1523	+ unsigned int node)
	1524	+{
	1525	+ unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
	1526	+ unsigned int rem = req->request.length % maxp;
	1527	+ unsigned int num_trbs = 1;
	1528	+
	1529	+ if (dwc3_needs_extra_trb(dep, req))
	1530	+ num_trbs++;
	1531	+
	1532	+ if (dwc3_calc_trbs_left(dep) < num_trbs)
	1533	+ return 0;
	1534	+
	1535	+ req->needs_extra_trb = num_trbs > 1;
	1536	+
	1537	+ /* Prepare a normal TRB */
	1538	+ if (req->direction \|\| req->request.length)
	1539	+ dwc3_prepare_one_trb(dep, req, entry_length,
	1540	+ req->needs_extra_trb, node, false, false);
	1541	+
	1542	+ /* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
	1543	+ if ((!req->direction && !req->request.length) \|\| req->needs_extra_trb)
	1544	+ dwc3_prepare_one_trb(dep, req,
	1545	+ req->direction ? 0 : maxp - rem,
	1546	+ false, 1, true, false);
	1547	+
	1548	+ return num_trbs;
	1549	+}
	1550	+
	1551	+static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1259	1552	struct dwc3_request *req)
1260	1553	{
1261	1554	struct scatterlist *sg = req->start_sg;
1262	1555	struct scatterlist *s;
1263	1556	int i;
1264	1557	unsigned int length = req->request.length;
1265		- unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1266		- unsigned int rem = length % maxp;
1267	1558	unsigned int remaining = req->request.num_mapped_sgs
1268	1559	- req->num_queued_sgs;
	1560	+ unsigned int num_trbs = req->num_trbs;
	1561	+ bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1269	1562
1270	1563	/*
1271	1564	* If we resume preparing the request, then get the remaining length of
..	..	@@ -1275,8 +1568,10 @@
1275	1568	length -= sg_dma_len(s);
1276	1569
1277	1570	for_each_sg(sg, s, remaining, i) {
	1571	+ unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1278	1572	unsigned int trb_length;
1279		- unsigned chain = true;
	1573	+ bool must_interrupt = false;
	1574	+ bool last_sg = false;
1280	1575
1281	1576	trb_length = min_t(unsigned int, length, sg_dma_len(s));
1282	1577
..	..	@@ -1290,56 +1585,28 @@
1290	1585	* mapped sg.
1291	1586	*/
1292	1587	if ((i == remaining - 1) \|\| !length)
1293		- chain = false;
	1588	+ last_sg = true;
1294	1589
1295		- if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1296		- struct dwc3 *dwc = dep->dwc;
1297		- struct dwc3_trb *trb;
	1590	+ if (!num_trbs_left)
	1591	+ break;
1298	1592
1299		- req->needs_extra_trb = true;
1300		-
1301		- /* prepare normal TRB */
1302		- dwc3_prepare_one_trb(dep, req, trb_length, true, i);
1303		-
1304		- /* Now prepare one extra TRB to align transfer size */
1305		- trb = &dep->trb_pool[dep->trb_enqueue];
1306		- req->num_trbs++;
1307		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1308		- maxp - rem, false, 1,
1309		- req->request.stream_id,
1310		- req->request.short_not_ok,
1311		- req->request.no_interrupt);
1312		- } else if (req->request.zero && req->request.length &&
1313		- !usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1314		- !rem && !chain) {
1315		- struct dwc3 *dwc = dep->dwc;
1316		- struct dwc3_trb *trb;
1317		-
1318		- req->needs_extra_trb = true;
1319		-
1320		- /* Prepare normal TRB */
1321		- dwc3_prepare_one_trb(dep, req, trb_length, true, i);
1322		-
1323		- /* Prepare one extra TRB to handle ZLP */
1324		- trb = &dep->trb_pool[dep->trb_enqueue];
1325		- req->num_trbs++;
1326		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1327		- !req->direction, 1,
1328		- req->request.stream_id,
1329		- req->request.short_not_ok,
1330		- req->request.no_interrupt);
1331		-
1332		- /* Prepare one more TRB to handle MPS alignment */
1333		- if (!req->direction) {
1334		- trb = &dep->trb_pool[dep->trb_enqueue];
1335		- req->num_trbs++;
1336		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp,
1337		- false, 1, req->request.stream_id,
1338		- req->request.short_not_ok,
1339		- req->request.no_interrupt);
1340		- }
	1593	+ if (last_sg) {
	1594	+ if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
	1595	+ break;
1341	1596	} else {
1342		- dwc3_prepare_one_trb(dep, req, trb_length, chain, i);
	1597	+ /*
	1598	+ * Look ahead to check if we have enough TRBs for the
	1599	+ * next SG entry. If not, set interrupt on this TRB to
	1600	+ * resume preparing the next SG entry when more TRBs are
	1601	+ * free.
	1602	+ */
	1603	+ if (num_trbs_left == 1 \|\| (needs_extra_trb &&
	1604	+ num_trbs_left <= 2 &&
	1605	+ sg_dma_len(sg_next(s)) >= length))
	1606	+ must_interrupt = true;
	1607	+
	1608	+ dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
	1609	+ must_interrupt);
1343	1610	}
1344	1611
1345	1612	/*
..	..	@@ -1349,7 +1616,7 @@
1349	1616	* we have free trbs we can continue queuing from where we
1350	1617	* previously stopped
1351	1618	*/
1352		- if (chain)
	1619	+ if (!last_sg)
1353	1620	req->start_sg = sg_next(s);
1354	1621
1355	1622	req->num_queued_sgs++;
..	..	@@ -1365,65 +1632,17 @@
1365	1632	break;
1366	1633	}
1367	1634
1368		- if (!dwc3_calc_trbs_left(dep))
	1635	+ if (must_interrupt)
1369	1636	break;
1370	1637	}
	1638	+
	1639	+ return req->num_trbs - num_trbs;
1371	1640	}
1372	1641
1373		-static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
	1642	+static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1374	1643	struct dwc3_request *req)
1375	1644	{
1376		- unsigned int length = req->request.length;
1377		- unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1378		- unsigned int rem = length % maxp;
1379		-
1380		- if ((!length \|\| rem) && usb_endpoint_dir_out(dep->endpoint.desc)) {
1381		- struct dwc3 *dwc = dep->dwc;
1382		- struct dwc3_trb *trb;
1383		-
1384		- req->needs_extra_trb = true;
1385		-
1386		- /* prepare normal TRB */
1387		- dwc3_prepare_one_trb(dep, req, length, true, 0);
1388		-
1389		- /* Now prepare one extra TRB to align transfer size */
1390		- trb = &dep->trb_pool[dep->trb_enqueue];
1391		- req->num_trbs++;
1392		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1393		- false, 1, req->request.stream_id,
1394		- req->request.short_not_ok,
1395		- req->request.no_interrupt);
1396		- } else if (req->request.zero && req->request.length &&
1397		- !usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1398		- (IS_ALIGNED(req->request.length, maxp))) {
1399		- struct dwc3 *dwc = dep->dwc;
1400		- struct dwc3_trb *trb;
1401		-
1402		- req->needs_extra_trb = true;
1403		-
1404		- /* prepare normal TRB */
1405		- dwc3_prepare_one_trb(dep, req, length, true, 0);
1406		-
1407		- /* Prepare one extra TRB to handle ZLP */
1408		- trb = &dep->trb_pool[dep->trb_enqueue];
1409		- req->num_trbs++;
1410		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1411		- !req->direction, 1, req->request.stream_id,
1412		- req->request.short_not_ok,
1413		- req->request.no_interrupt);
1414		-
1415		- /* Prepare one more TRB to handle MPS alignment for OUT */
1416		- if (!req->direction) {
1417		- trb = &dep->trb_pool[dep->trb_enqueue];
1418		- req->num_trbs++;
1419		- __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp,
1420		- false, 1, req->request.stream_id,
1421		- req->request.short_not_ok,
1422		- req->request.no_interrupt);
1423		- }
1424		- } else {
1425		- dwc3_prepare_one_trb(dep, req, length, false, 0);
1426		- }
	1645	+ return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1427	1646	}
1428	1647
1429	1648	/*
..	..	@@ -1433,10 +1652,13 @@
1433	1652	* The function goes through the requests list and sets up TRBs for the
1434	1653	* transfers. The function returns once there are no more TRBs available or
1435	1654	* it runs out of requests.
	1655	+ *
	1656	+ * Returns the number of TRBs prepared or negative errno.
1436	1657	*/
1437		-static void dwc3_prepare_trbs(struct dwc3_ep *dep)
	1658	+static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1438	1659	{
1439	1660	struct dwc3_request req, n;
	1661	+ int ret = 0;
1440	1662
1441	1663	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1442	1664
..	..	@@ -1451,35 +1673,58 @@
1451	1673	* break things.
1452	1674	*/
1453	1675	list_for_each_entry(req, &dep->started_list, list) {
1454		- if (req->num_pending_sgs > 0)
1455		- dwc3_prepare_one_trb_sg(dep, req);
	1676	+ if (req->num_pending_sgs > 0) {
	1677	+ ret = dwc3_prepare_trbs_sg(dep, req);
	1678	+ if (!ret \|\| req->num_pending_sgs)
	1679	+ return ret;
	1680	+ }
1456	1681
1457	1682	if (!dwc3_calc_trbs_left(dep))
1458		- return;
	1683	+ return ret;
	1684	+
	1685	+ /*
	1686	+ * Don't prepare beyond a transfer. In DWC_usb32, its transfer
	1687	+ * burst capability may try to read and use TRBs beyond the
	1688	+ * active transfer instead of stopping.
	1689	+ */
	1690	+ if (dep->stream_capable && req->request.is_last)
	1691	+ return ret;
1459	1692	}
1460	1693
1461	1694	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1462	1695	struct dwc3 *dwc = dep->dwc;
1463		- int ret;
1464	1696
1465	1697	ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1466	1698	dep->direction);
1467	1699	if (ret)
1468		- return;
	1700	+ return ret;
1469	1701
1470	1702	req->sg = req->request.sg;
1471	1703	req->start_sg = req->sg;
1472	1704	req->num_queued_sgs = 0;
1473	1705	req->num_pending_sgs = req->request.num_mapped_sgs;
1474	1706
1475		- if (req->num_pending_sgs > 0)
1476		- dwc3_prepare_one_trb_sg(dep, req);
1477		- else
1478		- dwc3_prepare_one_trb_linear(dep, req);
	1707	+ if (req->num_pending_sgs > 0) {
	1708	+ ret = dwc3_prepare_trbs_sg(dep, req);
	1709	+ if (req->num_pending_sgs)
	1710	+ return ret;
	1711	+ } else {
	1712	+ ret = dwc3_prepare_trbs_linear(dep, req);
	1713	+ }
1479	1714
1480		- if (!dwc3_calc_trbs_left(dep))
1481		- return;
	1715	+ if (!ret \|\| !dwc3_calc_trbs_left(dep))
	1716	+ return ret;
	1717	+
	1718	+ /*
	1719	+ * Don't prepare beyond a transfer. In DWC_usb32, its transfer
	1720	+ * burst capability may try to read and use TRBs beyond the
	1721	+ * active transfer instead of stopping.
	1722	+ */
	1723	+ if (dep->stream_capable && req->request.is_last)
	1724	+ return ret;
1482	1725	}
	1726	+
	1727	+ return ret;
1483	1728	}
1484	1729
1485	1730	static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
..	..	@@ -1492,12 +1737,24 @@
1492	1737	int ret;
1493	1738	u32 cmd;
1494	1739
1495		- if (!dwc3_calc_trbs_left(dep))
1496		- return 0;
	1740	+ /*
	1741	+ * Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
	1742	+ * This happens when we need to stop and restart a transfer such as in
	1743	+ * the case of reinitiating a stream or retrying an isoc transfer.
	1744	+ */
	1745	+ ret = dwc3_prepare_trbs(dep);
	1746	+ if (ret < 0)
	1747	+ return ret;
1497	1748
1498	1749	starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1499	1750
1500		- dwc3_prepare_trbs(dep);
	1751	+ /*
	1752	+ * If there's no new TRB prepared and we don't need to restart a
	1753	+ * transfer, there's no need to update the transfer.
	1754	+ */
	1755	+ if (!ret && !starting)
	1756	+ return ret;
	1757	+
1501	1758	req = next_request(&dep->started_list);
1502	1759	if (!req) {
1503	1760	dep->flags \|= DWC3_EP_PENDING_REQUEST;
..	..	@@ -1525,22 +1782,13 @@
1525	1782	if (ret < 0) {
1526	1783	struct dwc3_request *tmp;
1527	1784
1528		- /*
1529		- * Isochronous endpoints request needs to
1530		- * return directly and retry to transfer next
1531		- * time. Otherwise, it will fail to giveback
1532		- * the req to the udc gadget driver.
1533		- */
1534		- if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1535		- return ret;
1536		-
1537	1785	if (ret == -EAGAIN)
1538	1786	return ret;
1539	1787
1540	1788	dwc3_stop_active_transfer(dep, true, true);
1541	1789
1542	1790	list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1543		- dwc3_gadget_move_cancelled_request(req);
	1791	+ dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1544	1792
1545	1793	/* If ep isn't started, then there's no end transfer pending */
1546	1794	if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
..	..	@@ -1548,6 +1796,9 @@
1548	1796
1549	1797	return ret;
1550	1798	}
	1799	+
	1800	+ if (dep->stream_capable && req->request.is_last)
	1801	+ dep->flags \|= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1551	1802
1552	1803	return 0;
1553	1804	}
..	..	@@ -1558,6 +1809,55 @@
1558	1809
1559	1810	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1560	1811	return DWC3_DSTS_SOFFN(reg);
	1812	+}
	1813	+
	1814	+/**
	1815	+ * __dwc3_stop_active_transfer - stop the current active transfer
	1816	+ * @dep: isoc endpoint
	1817	+ * @force: set forcerm bit in the command
	1818	+ * @interrupt: command complete interrupt after End Transfer command
	1819	+ *
	1820	+ * When setting force, the ForceRM bit will be set. In that case
	1821	+ * the controller won't update the TRB progress on command
	1822	+ * completion. It also won't clear the HWO bit in the TRB.
	1823	+ * The command will also not complete immediately in that case.
	1824	+ */
	1825	+static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
	1826	+{
	1827	+ struct dwc3 *dwc = dep->dwc;
	1828	+ struct dwc3_gadget_ep_cmd_params params;
	1829	+ u32 cmd;
	1830	+ int ret;
	1831	+
	1832	+ cmd = DWC3_DEPCMD_ENDTRANSFER;
	1833	+ cmd \|= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
	1834	+ cmd \|= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
	1835	+ cmd \|= DWC3_DEPCMD_PARAM(dep->resource_index);
	1836	+ memset(&params, 0, sizeof(params));
	1837	+ ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
	1838	+ /*
	1839	+ * If the End Transfer command was timed out while the device is
	1840	+ * not in SETUP phase, it's possible that an incoming Setup packet
	1841	+ * may prevent the command's completion. Let's retry when the
	1842	+ * ep0state returns to EP0_SETUP_PHASE.
	1843	+ */
	1844	+ if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
	1845	+ dep->flags \|= DWC3_EP_DELAY_STOP;
	1846	+ return 0;
	1847	+ }
	1848	+ WARN_ON_ONCE(ret);
	1849	+ dep->resource_index = 0;
	1850	+
	1851	+ if (!interrupt) {
	1852	+ if (!DWC3_IP_IS(DWC3) \|\| DWC3_VER_IS_PRIOR(DWC3, 310A))
	1853	+ mdelay(1);
	1854	+ dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
	1855	+ } else {
	1856	+ dep->flags \|= DWC3_EP_END_TRANSFER_PENDING;
	1857	+ }
	1858	+
	1859	+ dep->flags &= ~DWC3_EP_DELAY_STOP;
	1860	+ return ret;
1561	1861	}
1562	1862
1563	1863	/**
..	..	@@ -1617,7 +1917,7 @@
1617	1917	* Check if we can start isoc transfer on the next interval or
1618	1918	* 4 uframes in the future with BIT[15:14] as dep->combo_num
1619	1919	*/
1620		- test_frame_number = dep->frame_number & 0x3fff;
	1920	+ test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1621	1921	test_frame_number \|= dep->combo_num << 14;
1622	1922	test_frame_number += max_t(u32, 4, dep->interval);
1623	1923
..	..	@@ -1664,7 +1964,7 @@
1664	1964	else if (test0 && test1)
1665	1965	dep->combo_num = 0;
1666	1966
1667		- dep->frame_number &= 0x3fff;
	1967	+ dep->frame_number &= DWC3_FRNUMBER_MASK;
1668	1968	dep->frame_number \|= dep->combo_num << 14;
1669	1969	dep->frame_number += max_t(u32, 4, dep->interval);
1670	1970
..	..	@@ -1682,26 +1982,24 @@
1682	1982	int ret;
1683	1983	int i;
1684	1984
1685		- if (list_empty(&dep->pending_list)) {
	1985	+ if (list_empty(&dep->pending_list) &&
	1986	+ list_empty(&dep->started_list)) {
1686	1987	dep->flags \|= DWC3_EP_PENDING_REQUEST;
1687	1988	return -EAGAIN;
1688	1989	}
1689	1990
1690		- if (!dwc->dis_start_transfer_quirk && dwc3_is_usb31(dwc) &&
1691		- (dwc->revision <= DWC3_USB31_REVISION_160A \|\|
1692		- (dwc->revision == DWC3_USB31_REVISION_170A &&
1693		- dwc->version_type >= DWC31_VERSIONTYPE_EA01 &&
1694		- dwc->version_type <= DWC31_VERSIONTYPE_EA06))) {
1695		-
1696		- if (dwc->gadget.speed <= USB_SPEED_HIGH && dep->direction)
	1991	+ if (!dwc->dis_start_transfer_quirk &&
	1992	+ (DWC3_VER_IS_PRIOR(DWC31, 170A) \|\|
	1993	+ DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
	1994	+ if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1697	1995	return dwc3_gadget_start_isoc_quirk(dep);
1698	1996	}
1699	1997
1700	1998	if (desc->bInterval <= 14 &&
1701		- dwc->gadget.speed >= USB_SPEED_HIGH) {
	1999	+ dwc->gadget->speed >= USB_SPEED_HIGH) {
1702	2000	u32 frame = __dwc3_gadget_get_frame(dwc);
1703	2001	bool rollover = frame <
1704		- (dep->frame_number & 0x3fff);
	2002	+ (dep->frame_number & DWC3_FRNUMBER_MASK);
1705	2003
1706	2004	/*
1707	2005	* frame_number is set from XferNotReady and may be already
..	..	@@ -1712,54 +2010,46 @@
1712	2010	* rollover has happened since XferNotReady.
1713	2011	*/
1714	2012
1715		- dep->frame_number = (dep->frame_number & ~0x3fff) \|
	2013	+ dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) \|
1716	2014	frame;
1717	2015	if (rollover)
1718	2016	dep->frame_number += BIT(14);
1719	2017	}
1720	2018
1721	2019	for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1722		- dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1);
	2020	+ int future_interval = i + 1;
	2021	+
	2022	+ /* Give the controller at least 500us to schedule transfers */
	2023	+ if (desc->bInterval < 3)
	2024	+ future_interval += 3 - desc->bInterval;
	2025	+
	2026	+ dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1723	2027
1724	2028	ret = __dwc3_gadget_kick_transfer(dep);
1725	2029	if (ret != -EAGAIN)
1726	2030	break;
1727	2031	}
1728	2032
1729		- return ret;
1730		-}
1731		-
1732		-static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep dep, struct dwc3_request req)
1733		-{
1734		- int i;
1735		-
1736	2033	/*
1737		- * If request was already started, this means we had to
1738		- * stop the transfer. With that we also need to ignore
1739		- * all TRBs used by the request, however TRBs can only
1740		- * be modified after completion of END_TRANSFER
1741		- * command. So what we do here is that we wait for
1742		- * END_TRANSFER completion and only after that, we jump
1743		- * over TRBs by clearing HWO and incrementing dequeue
1744		- * pointer.
	2034	+ * After a number of unsuccessful start attempts due to bus-expiry
	2035	+ * status, issue END_TRANSFER command and retry on the next XferNotReady
	2036	+ * event.
1745	2037	*/
1746		- for (i = 0; i < req->num_trbs; i++) {
1747		- struct dwc3_trb *trb;
1748		-
1749		- trb = &dep->trb_pool[dep->trb_dequeue];
1750		- trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1751		- dwc3_ep_inc_deq(dep);
	2038	+ if (ret == -EAGAIN) {
	2039	+ ret = __dwc3_stop_active_transfer(dep, false, true);
	2040	+ if (ret)
	2041	+ dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
1752	2042	}
1753	2043
1754		- req->num_trbs = 0;
	2044	+ return ret;
1755	2045	}
1756	2046
1757	2047	static int __dwc3_gadget_ep_queue(struct dwc3_ep dep, struct dwc3_request req)
1758	2048	{
1759	2049	struct dwc3 *dwc = dep->dwc;
1760	2050
1761		- if (!dep->endpoint.desc) {
1762		- dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
	2051	+ if (!dep->endpoint.desc \|\| !dwc->pullups_connected \|\| !dwc->connected) {
	2052	+ dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1763	2053	dep->name);
1764	2054	return -ESHUTDOWN;
1765	2055	}
..	..	@@ -1770,14 +2060,8 @@
1770	2060
1771	2061	if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1772	2062	"%s: request %pK already in flight\n",
1773		- dep->name, &req->request)) {
1774		- if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1775		- dwc3_gadget_ep_skip_trbs(dep, req);
1776		- req->status = DWC3_REQUEST_STATUS_COMPLETED;
1777		- dwc3_gadget_del_and_unmap_request(dep, req, -EINVAL);
1778		- }
	2063	+ dep->name, &req->request))
1779	2064	return -EINVAL;
1780		- }
1781	2065
1782	2066	pm_runtime_get(dwc->dev);
1783	2067
..	..	@@ -1789,8 +2073,17 @@
1789	2073	list_add_tail(&req->list, &dep->pending_list);
1790	2074	req->status = DWC3_REQUEST_STATUS_QUEUED;
1791	2075
1792		- /* Start the transfer only after the END_TRANSFER is completed */
1793		- if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) {
	2076	+ if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
	2077	+ return 0;
	2078	+
	2079	+ /*
	2080	+ * Start the transfer only after the END_TRANSFER is completed
	2081	+ * and endpoint STALL is cleared.
	2082	+ */
	2083	+ if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) \|\|
	2084	+ (dep->flags & DWC3_EP_WEDGE) \|\|
	2085	+ (dep->flags & DWC3_EP_DELAY_STOP) \|\|
	2086	+ (dep->flags & DWC3_EP_STALL)) {
1794	2087	dep->flags \|= DWC3_EP_DELAY_START;
1795	2088	return 0;
1796	2089	}
..	..	@@ -1804,14 +2097,11 @@
1804	2097	* errors which will force us issue EndTransfer command.
1805	2098	*/
1806	2099	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1807		- if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1808		- !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1809		- return 0;
1810		-
1811		- if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1812		- if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
	2100	+ if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
	2101	+ if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1813	2102	return __dwc3_gadget_start_isoc(dep);
1814		- }
	2103	+
	2104	+ return 0;
1815	2105	}
1816	2106	}
1817	2107
..	..	@@ -1838,14 +2128,64 @@
1838	2128	return ret;
1839	2129	}
1840	2130
	2131	+static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep dep, struct dwc3_request req)
	2132	+{
	2133	+ int i;
	2134	+
	2135	+ /* If req->trb is not set, then the request has not started */
	2136	+ if (!req->trb)
	2137	+ return;
	2138	+
	2139	+ /*
	2140	+ * If request was already started, this means we had to
	2141	+ * stop the transfer. With that we also need to ignore
	2142	+ * all TRBs used by the request, however TRBs can only
	2143	+ * be modified after completion of END_TRANSFER
	2144	+ * command. So what we do here is that we wait for
	2145	+ * END_TRANSFER completion and only after that, we jump
	2146	+ * over TRBs by clearing HWO and incrementing dequeue
	2147	+ * pointer.
	2148	+ */
	2149	+ for (i = 0; i < req->num_trbs; i++) {
	2150	+ struct dwc3_trb *trb;
	2151	+
	2152	+ trb = &dep->trb_pool[dep->trb_dequeue];
	2153	+ trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
	2154	+ dwc3_ep_inc_deq(dep);
	2155	+ }
	2156	+
	2157	+ req->num_trbs = 0;
	2158	+}
	2159	+
1841	2160	static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
1842	2161	{
1843	2162	struct dwc3_request *req;
1844		- struct dwc3_request *tmp;
	2163	+ struct dwc3 *dwc = dep->dwc;
1845	2164
1846		- list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) {
	2165	+ while (!list_empty(&dep->cancelled_list)) {
	2166	+ req = next_request(&dep->cancelled_list);
1847	2167	dwc3_gadget_ep_skip_trbs(dep, req);
1848		- dwc3_gadget_giveback(dep, req, -ECONNRESET);
	2168	+ switch (req->status) {
	2169	+ case DWC3_REQUEST_STATUS_DISCONNECTED:
	2170	+ dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
	2171	+ break;
	2172	+ case DWC3_REQUEST_STATUS_DEQUEUED:
	2173	+ dwc3_gadget_giveback(dep, req, -ECONNRESET);
	2174	+ break;
	2175	+ case DWC3_REQUEST_STATUS_STALLED:
	2176	+ dwc3_gadget_giveback(dep, req, -EPIPE);
	2177	+ break;
	2178	+ default:
	2179	+ dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
	2180	+ dwc3_gadget_giveback(dep, req, -ECONNRESET);
	2181	+ break;
	2182	+ }
	2183	+ /*
	2184	+ * The endpoint is disabled, let the dwc3_remove_requests()
	2185	+ * handle the cleanup.
	2186	+ */
	2187	+ if (!dep->endpoint.desc)
	2188	+ break;
1849	2189	}
1850	2190	}
1851	2191
..	..	@@ -1865,40 +2205,45 @@
1865	2205
1866	2206	spin_lock_irqsave(&dwc->lock, flags);
1867	2207
1868		- list_for_each_entry(r, &dep->pending_list, list) {
	2208	+ list_for_each_entry(r, &dep->cancelled_list, list) {
1869	2209	if (r == req)
1870		- break;
	2210	+ goto out;
1871	2211	}
1872	2212
1873		- if (r != req) {
1874		- list_for_each_entry(r, &dep->started_list, list) {
1875		- if (r == req)
1876		- break;
	2213	+ list_for_each_entry(r, &dep->pending_list, list) {
	2214	+ if (r == req) {
	2215	+ dwc3_gadget_ep_skip_trbs(dep, req);
	2216	+ dwc3_gadget_giveback(dep, req, -ECONNRESET);
	2217	+ goto out;
1877	2218	}
	2219	+ }
	2220	+
	2221	+ list_for_each_entry(r, &dep->started_list, list) {
1878	2222	if (r == req) {
1879	2223	/* wait until it is processed */
1880	2224	dwc3_stop_active_transfer(dep, true, true);
1881	2225
1882		- if (!r->trb)
1883		- goto out0;
	2226	+ /*
	2227	+ * Remove any started request if the transfer is
	2228	+ * cancelled.
	2229	+ */
	2230	+ dwc3_gadget_move_cancelled_request(r, DWC3_REQUEST_STATUS_DEQUEUED);
1884	2231
1885		- dwc3_gadget_move_cancelled_request(req);
1886		- if (dep->flags & DWC3_EP_TRANSFER_STARTED)
1887		- goto out0;
1888		- else
1889		- goto out1;
	2232	+ dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
	2233	+
	2234	+ if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
	2235	+ dwc3_gadget_ep_skip_trbs(dep, req);
	2236	+ dwc3_gadget_giveback(dep, req, -ECONNRESET);
	2237	+ }
	2238	+
	2239	+ goto out;
1890	2240	}
1891		- dev_err(dwc->dev, "request %pK was not queued to %s\n",
1892		- request, ep->name);
1893		- ret = -EINVAL;
1894		- goto out0;
1895	2241	}
1896	2242
1897		-out1:
1898		- dwc3_gadget_ep_skip_trbs(dep, req);
1899		- dwc3_gadget_giveback(dep, req, -ECONNRESET);
1900		-
1901		-out0:
	2243	+ dev_err(dwc->dev, "request %pK was not queued to %s\n",
	2244	+ request, ep->name);
	2245	+ ret = -EINVAL;
	2246	+out:
1902	2247	spin_unlock_irqrestore(&dwc->lock, flags);
1903	2248
1904	2249	return ret;
..	..	@@ -1909,6 +2254,7 @@
1909	2254	struct dwc3_gadget_ep_cmd_params params;
1910	2255	struct dwc3 *dwc = dep->dwc;
1911	2256	int ret;
	2257	+ struct dwc3_vendor *vdwc = container_of(dwc, struct dwc3_vendor, dwc);
1912	2258
1913	2259	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1914	2260	dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
..	..	@@ -1920,8 +2266,8 @@
1920	2266	if (value) {
1921	2267	struct dwc3_trb *trb;
1922	2268
1923		- unsigned transfer_in_flight;
1924		- unsigned started;
	2269	+ unsigned int transfer_in_flight;
	2270	+ unsigned int started;
1925	2271
1926	2272	if (dep->number > 1)
1927	2273	trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
..	..	@@ -1944,13 +2290,44 @@
1944	2290	else
1945	2291	dep->flags \|= DWC3_EP_STALL;
1946	2292	} else {
	2293	+ /*
	2294	+ * Don't issue CLEAR_STALL command to control endpoints. The
	2295	+ * controller automatically clears the STALL when it receives
	2296	+ * the SETUP token.
	2297	+ */
	2298	+ if (dep->number <= 1) {
	2299	+ dep->flags &= ~(DWC3_EP_STALL \| DWC3_EP_WEDGE);
	2300	+ return 0;
	2301	+ }
	2302	+
	2303	+ dwc3_stop_active_transfer(dep, true, true);
	2304	+
	2305	+ if (!list_empty(&dep->started_list))
	2306	+ dep->flags \|= DWC3_EP_DELAY_START;
	2307	+
	2308	+ if (dep->flags & DWC3_EP_END_TRANSFER_PENDING \|\|
	2309	+ (dep->flags & DWC3_EP_DELAY_STOP)) {
	2310	+ dep->flags \|= DWC3_EP_PENDING_CLEAR_STALL;
	2311	+ if (protocol)
	2312	+ vdwc->clear_stall_protocol = dep->number;
	2313	+
	2314	+ return 0;
	2315	+ }
1947	2316
1948	2317	ret = dwc3_send_clear_stall_ep_cmd(dep);
1949		- if (ret)
	2318	+ if (ret) {
1950	2319	dev_err(dwc->dev, "failed to clear STALL on %s\n",
1951	2320	dep->name);
1952		- else
1953		- dep->flags &= ~(DWC3_EP_STALL \| DWC3_EP_WEDGE);
	2321	+ return ret;
	2322	+ }
	2323	+
	2324	+ dep->flags &= ~(DWC3_EP_STALL \| DWC3_EP_WEDGE);
	2325	+
	2326	+ if ((dep->flags & DWC3_EP_DELAY_START) &&
	2327	+ !usb_endpoint_xfer_isoc(dep->endpoint.desc))
	2328	+ __dwc3_gadget_kick_transfer(dep);
	2329	+
	2330	+ dep->flags &= ~DWC3_EP_DELAY_START;
1954	2331	}
1955	2332
1956	2333	return ret;
..	..	@@ -2050,7 +2427,6 @@
2050	2427	link_state = DWC3_DSTS_USBLNKST(reg);
2051	2428
2052	2429	switch (link_state) {
2053		- case DWC3_LINK_STATE_U0:
2054	2430	case DWC3_LINK_STATE_RESET:
2055	2431	case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
2056	2432	case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
..	..	@@ -2062,15 +2438,6 @@
2062	2438	return -EINVAL;
2063	2439	}
2064	2440
2065		- /*
2066		- * dwc3 gadget wakeup from host resume signal
2067		- * when the whole system enter suspend.
2068		- */
2069		- if (link_state == DWC3_LINK_STATE_U0) {
2070		- dwc->link_state = link_state;
2071		- return 0;
2072		- }
2073		-
2074	2441	ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2075	2442	if (ret < 0) {
2076	2443	dev_err(dwc->dev, "failed to put link in Recovery\n");
..	..	@@ -2078,7 +2445,7 @@
2078	2445	}
2079	2446
2080	2447	/* Recent versions do this automatically */
2081		- if (dwc->revision < DWC3_REVISION_194A) {
	2448	+ if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2082	2449	/* write zeroes to Link Change Request */
2083	2450	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2084	2451	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
..	..	@@ -2130,28 +2497,140 @@
2130	2497	return 0;
2131	2498	}
2132	2499
	2500	+static void dwc3_stop_active_transfers(struct dwc3 *dwc)
	2501	+{
	2502	+ u32 epnum;
	2503	+
	2504	+ for (epnum = 2; epnum < dwc->num_eps; epnum++) {
	2505	+ struct dwc3_ep *dep;
	2506	+
	2507	+ dep = dwc->eps[epnum];
	2508	+ if (!dep)
	2509	+ continue;
	2510	+
	2511	+ dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
	2512	+ }
	2513	+}
	2514	+
	2515	+static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
	2516	+{
	2517	+ enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate;
	2518	+ u32 reg;
	2519	+
	2520	+ if (ssp_rate == USB_SSP_GEN_UNKNOWN)
	2521	+ ssp_rate = dwc->max_ssp_rate;
	2522	+
	2523	+ reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	2524	+ reg &= ~DWC3_DCFG_SPEED_MASK;
	2525	+ reg &= ~DWC3_DCFG_NUMLANES(~0);
	2526	+
	2527	+ if (ssp_rate == USB_SSP_GEN_1x2)
	2528	+ reg \|= DWC3_DCFG_SUPERSPEED;
	2529	+ else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
	2530	+ reg \|= DWC3_DCFG_SUPERSPEED_PLUS;
	2531	+
	2532	+ if (ssp_rate != USB_SSP_GEN_2x1 &&
	2533	+ dwc->max_ssp_rate != USB_SSP_GEN_2x1)
	2534	+ reg \|= DWC3_DCFG_NUMLANES(1);
	2535	+
	2536	+ dwc3_writel(dwc->regs, DWC3_DCFG, reg);
	2537	+}
	2538	+
	2539	+static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
	2540	+{
	2541	+ enum usb_device_speed speed;
	2542	+ u32 reg;
	2543	+
	2544	+ speed = dwc->gadget_max_speed;
	2545	+ if (speed == USB_SPEED_UNKNOWN \|\| speed > dwc->maximum_speed)
	2546	+ speed = dwc->maximum_speed;
	2547	+
	2548	+ if (speed == USB_SPEED_SUPER_PLUS &&
	2549	+ DWC3_IP_IS(DWC32)) {
	2550	+ __dwc3_gadget_set_ssp_rate(dwc);
	2551	+ return;
	2552	+ }
	2553	+
	2554	+ reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	2555	+ reg &= ~(DWC3_DCFG_SPEED_MASK);
	2556	+
	2557	+ /*
	2558	+ * WORKAROUND: DWC3 revision < 2.20a have an issue
	2559	+ * which would cause metastability state on Run/Stop
	2560	+ * bit if we try to force the IP to USB2-only mode.
	2561	+ *
	2562	+ * Because of that, we cannot configure the IP to any
	2563	+ * speed other than the SuperSpeed
	2564	+ *
	2565	+ * Refers to:
	2566	+ *
	2567	+ * STAR#9000525659: Clock Domain Crossing on DCTL in
	2568	+ * USB 2.0 Mode
	2569	+ */
	2570	+ if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
	2571	+ !dwc->dis_metastability_quirk) {
	2572	+ reg \|= DWC3_DCFG_SUPERSPEED;
	2573	+ } else {
	2574	+ switch (speed) {
	2575	+ case USB_SPEED_LOW:
	2576	+ reg \|= DWC3_DCFG_LOWSPEED;
	2577	+ break;
	2578	+ case USB_SPEED_FULL:
	2579	+ reg \|= DWC3_DCFG_FULLSPEED;
	2580	+ break;
	2581	+ case USB_SPEED_HIGH:
	2582	+ reg \|= DWC3_DCFG_HIGHSPEED;
	2583	+ break;
	2584	+ case USB_SPEED_SUPER:
	2585	+ reg \|= DWC3_DCFG_SUPERSPEED;
	2586	+ break;
	2587	+ case USB_SPEED_SUPER_PLUS:
	2588	+ if (DWC3_IP_IS(DWC3))
	2589	+ reg \|= DWC3_DCFG_SUPERSPEED;
	2590	+ else
	2591	+ reg \|= DWC3_DCFG_SUPERSPEED_PLUS;
	2592	+ break;
	2593	+ default:
	2594	+ dev_err(dwc->dev, "invalid speed (%d)\n", speed);
	2595	+
	2596	+ if (DWC3_IP_IS(DWC3))
	2597	+ reg \|= DWC3_DCFG_SUPERSPEED;
	2598	+ else
	2599	+ reg \|= DWC3_DCFG_SUPERSPEED_PLUS;
	2600	+ }
	2601	+ }
	2602	+
	2603	+ if (DWC3_IP_IS(DWC32) &&
	2604	+ speed > USB_SPEED_UNKNOWN &&
	2605	+ speed < USB_SPEED_SUPER_PLUS)
	2606	+ reg &= ~DWC3_DCFG_NUMLANES(~0);
	2607	+
	2608	+ dwc3_writel(dwc->regs, DWC3_DCFG, reg);
	2609	+}
	2610	+
2133	2611	static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
2134	2612	{
2135	2613	u32 reg;
2136		- u32 timeout = 500;
	2614	+ u32 timeout = 2000;
2137	2615
2138	2616	if (pm_runtime_suspended(dwc->dev))
2139	2617	return 0;
2140	2618
2141	2619	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2142	2620	if (is_on) {
2143		- if (dwc->revision <= DWC3_REVISION_187A) {
	2621	+ if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2144	2622	reg &= ~DWC3_DCTL_TRGTULST_MASK;
2145	2623	reg \|= DWC3_DCTL_TRGTULST_RX_DET;
2146	2624	}
2147	2625
2148		- if (dwc->revision >= DWC3_REVISION_194A)
	2626	+ if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2149	2627	reg &= ~DWC3_DCTL_KEEP_CONNECT;
2150	2628	reg \|= DWC3_DCTL_RUN_STOP;
2151	2629
2152	2630	if (dwc->has_hibernation)
2153	2631	reg \|= DWC3_DCTL_KEEP_CONNECT;
2154	2632
	2633	+ __dwc3_gadget_set_speed(dwc);
2155	2634	dwc->pullups_connected = true;
2156	2635	} else {
2157	2636	reg &= ~DWC3_DCTL_RUN_STOP;
..	..	@@ -2162,9 +2641,10 @@
2162	2641	dwc->pullups_connected = false;
2163	2642	}
2164	2643
2165		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	2644	+ dwc3_gadget_dctl_write_safe(dwc, reg);
2166	2645
2167	2646	do {
	2647	+ usleep_range(1000, 2000);
2168	2648	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2169	2649	reg &= DWC3_DSTS_DEVCTRLHLT;
2170	2650	} while (--timeout && !(!is_on ^ !reg));
..	..	@@ -2175,42 +2655,156 @@
2175	2655	return 0;
2176	2656	}
2177	2657
2178		-static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2179		-{
2180		- struct dwc3 *dwc = gadget_to_dwc(g);
2181		- unsigned long flags;
2182		- int ret;
	2658	+static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
	2659	+static void __dwc3_gadget_stop(struct dwc3 *dwc);
	2660	+static int __dwc3_gadget_start(struct dwc3 *dwc);
2183	2661
2184		- is_on = !!is_on;
	2662	+static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
	2663	+{
	2664	+ unsigned long flags;
	2665	+ int ret;
	2666	+
	2667	+ spin_lock_irqsave(&dwc->lock, flags);
	2668	+ dwc->connected = false;
	2669	+
	2670	+ /*
	2671	+ * Attempt to end pending SETUP status phase, and not wait for the
	2672	+ * function to do so.
	2673	+ */
	2674	+ if (dwc->delayed_status)
	2675	+ dwc3_ep0_send_delayed_status(dwc);
	2676	+
	2677	+ /*
	2678	+ * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
	2679	+ * Section 4.1.8 Table 4-7, it states that for a device-initiated
	2680	+ * disconnect, the SW needs to ensure that it sends "a DEPENDXFER
	2681	+ * command for any active transfers" before clearing the RunStop
	2682	+ * bit.
	2683	+ */
	2684	+ dwc3_stop_active_transfers(dwc);
	2685	+ spin_unlock_irqrestore(&dwc->lock, flags);
2185	2686
2186	2687	/*
2187	2688	* Per databook, when we want to stop the gadget, if a control transfer
2188	2689	* is still in process, complete it and get the core into setup phase.
	2690	+ * In case the host is unresponsive to a SETUP transaction, forcefully
	2691	+ * stall the transfer, and move back to the SETUP phase, so that any
	2692	+ * pending endxfers can be executed.
2189	2693	*/
2190		- if (!is_on && dwc->ep0state != EP0_SETUP_PHASE &&
	2694	+ if (dwc->ep0state != EP0_SETUP_PHASE &&
2191	2695	dwc->ep0state != EP0_UNCONNECTED) {
2192	2696	reinit_completion(&dwc->ep0_in_setup);
2193	2697
2194	2698	ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2195	2699	msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2196		- if (ret == 0)
2197		- dev_warn(dwc->dev, "timed out waiting for SETUP phase\n");
	2700	+ if (ret == 0) {
	2701	+ unsigned int dir;
	2702	+
	2703	+ dev_warn(dwc->dev, "wait for SETUP phase timed out\n");
	2704	+ spin_lock_irqsave(&dwc->lock, flags);
	2705	+ dir = !!dwc->ep0_expect_in;
	2706	+ if (dwc->ep0state == EP0_DATA_PHASE)
	2707	+ dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
	2708	+ else
	2709	+ dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
	2710	+ dwc3_ep0_stall_and_restart(dwc);
	2711	+ spin_unlock_irqrestore(&dwc->lock, flags);
	2712	+ }
2198	2713	}
2199	2714
	2715	+ /*
	2716	+ * Note: if the GEVNTCOUNT indicates events in the event buffer, the
	2717	+ * driver needs to acknowledge them before the controller can halt.
	2718	+ * Simply let the interrupt handler acknowledges and handle the
	2719	+ * remaining event generated by the controller while polling for
	2720	+ * DSTS.DEVCTLHLT.
	2721	+ */
	2722	+ ret = dwc3_gadget_run_stop(dwc, false, false);
	2723	+
	2724	+ /*
	2725	+ * Stop the gadget after controller is halted, so that if needed, the
	2726	+ * events to update EP0 state can still occur while the run/stop
	2727	+ * routine polls for the halted state. DEVTEN is cleared as part of
	2728	+ * gadget stop.
	2729	+ */
2200	2730	spin_lock_irqsave(&dwc->lock, flags);
2201		- ret = dwc3_gadget_run_stop(dwc, is_on, false);
	2731	+ __dwc3_gadget_stop(dwc);
2202	2732	spin_unlock_irqrestore(&dwc->lock, flags);
2203	2733
2204	2734	return ret;
2205	2735	}
2206	2736
2207		-void dwc3_gadget_enable_irq(struct dwc3 *dwc)
	2737	+static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
	2738	+{
	2739	+ struct dwc3 *dwc = gadget_to_dwc(g);
	2740	+ struct dwc3_vendor *vdwc = container_of(dwc, struct dwc3_vendor, dwc);
	2741	+ int ret;
	2742	+
	2743	+ is_on = !!is_on;
	2744	+
	2745	+ vdwc->softconnect = is_on;
	2746	+
	2747	+ /*
	2748	+ * Avoid issuing a runtime resume if the device is already in the
	2749	+ * suspended state during gadget disconnect. DWC3 gadget was already
	2750	+ * halted/stopped during runtime suspend.
	2751	+ */
	2752	+ if (!is_on) {
	2753	+ pm_runtime_barrier(dwc->dev);
	2754	+ if (pm_runtime_suspended(dwc->dev))
	2755	+ return 0;
	2756	+ }
	2757	+
	2758	+ /*
	2759	+ * Check the return value for successful resume, or error. For a
	2760	+ * successful resume, the DWC3 runtime PM resume routine will handle
	2761	+ * the run stop sequence, so avoid duplicate operations here.
	2762	+ */
	2763	+ ret = pm_runtime_get_sync(dwc->dev);
	2764	+ if (!ret \|\| ret < 0) {
	2765	+ pm_runtime_put(dwc->dev);
	2766	+ if (ret < 0)
	2767	+ pm_runtime_set_suspended(dwc->dev);
	2768	+ return ret;
	2769	+ }
	2770	+
	2771	+ if (dwc->pullups_connected == is_on) {
	2772	+ pm_runtime_put(dwc->dev);
	2773	+ return 0;
	2774	+ }
	2775	+
	2776	+ synchronize_irq(dwc->irq_gadget);
	2777	+
	2778	+ if (!is_on) {
	2779	+ ret = dwc3_gadget_soft_disconnect(dwc);
	2780	+ } else {
	2781	+ /*
	2782	+ * In the Synopsys DWC_usb31 1.90a programming guide section
	2783	+ * 4.1.9, it specifies that for a reconnect after a
	2784	+ * device-initiated disconnect requires a core soft reset
	2785	+ * (DCTL.CSftRst) before enabling the run/stop bit.
	2786	+ */
	2787	+ ret = dwc3_core_soft_reset(dwc);
	2788	+ if (ret)
	2789	+ goto done;
	2790	+
	2791	+ dwc3_event_buffers_setup(dwc);
	2792	+ __dwc3_gadget_start(dwc);
	2793	+ ret = dwc3_gadget_run_stop(dwc, true, false);
	2794	+ }
	2795	+
	2796	+done:
	2797	+ pm_runtime_put(dwc->dev);
	2798	+
	2799	+ return ret;
	2800	+}
	2801	+
	2802	+static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2208	2803	{
2209	2804	u32 reg;
2210	2805
2211	2806	/* Enable all but Start and End of Frame IRQs */
2212		- reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN \|
2213		- DWC3_DEVTEN_EVNTOVERFLOWEN \|
	2807	+ reg = (DWC3_DEVTEN_EVNTOVERFLOWEN \|
2214	2808	DWC3_DEVTEN_CMDCMPLTEN \|
2215	2809	DWC3_DEVTEN_ERRTICERREN \|
2216	2810	DWC3_DEVTEN_WKUPEVTEN \|
..	..	@@ -2218,17 +2812,17 @@
2218	2812	DWC3_DEVTEN_USBRSTEN \|
2219	2813	DWC3_DEVTEN_DISCONNEVTEN);
2220	2814
2221		- if (dwc->revision < DWC3_REVISION_250A)
	2815	+ if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2222	2816	reg \|= DWC3_DEVTEN_ULSTCNGEN;
2223	2817
2224	2818	/* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2225		- if (dwc->revision >= DWC3_REVISION_230A)
2226		- reg \|= DWC3_DEVTEN_EOPFEN;
	2819	+ if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
	2820	+ reg \|= DWC3_DEVTEN_U3L2L1SUSPEN;
2227	2821
2228	2822	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2229	2823	}
2230	2824
2231		-void dwc3_gadget_disable_irq(struct dwc3 *dwc)
	2825	+static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2232	2826	{
2233	2827	/* mask all interrupts */
2234	2828	dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
..	..	@@ -2266,7 +2860,7 @@
2266	2860	u32 reg;
2267	2861
2268	2862	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2269		- mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
	2863	+ mdwidth = dwc3_mdwidth(dwc);
2270	2864
2271	2865	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2272	2866	nump = min_t(u32, nump, 16);
..	..	@@ -2283,6 +2877,15 @@
2283	2877	struct dwc3_ep *dep;
2284	2878	int ret = 0;
2285	2879	u32 reg;
	2880	+
	2881	+ /*
	2882	+ * If the DWC3 is in runtime suspend, the clocks maybe
	2883	+ * disabled, so avoid enable the DWC3 endpoints here.
	2884	+ * The DWC3 runtime PM resume routine will handle the
	2885	+ * gadget start sequence.
	2886	+ */
	2887	+ if (pm_runtime_suspended(dwc->dev))
	2888	+ return ret;
2286	2889
2287	2890	/*
2288	2891	* Use IMOD if enabled via dwc->imod_interval. Otherwise, if
..	..	@@ -2303,19 +2906,31 @@
2303	2906	* bursts of data without going through any sort of endpoint throttling.
2304	2907	*/
2305	2908	reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2306		- if (dwc3_is_usb31(dwc))
2307		- reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2308		- else
	2909	+ if (DWC3_IP_IS(DWC3))
2309	2910	reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
	2911	+ else
	2912	+ reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2310	2913
2311	2914	dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2312	2915
2313	2916	dwc3_gadget_setup_nump(dwc);
2314	2917
	2918	+ /*
	2919	+ * Currently the controller handles single stream only. So, Ignore
	2920	+ * Packet Pending bit for stream selection and don't search for another
	2921	+ * stream if the host sends Data Packet with PP=0 (for OUT direction) or
	2922	+ * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
	2923	+ * the stream performance.
	2924	+ */
	2925	+ reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	2926	+ reg \|= DWC3_DCFG_IGNSTRMPP;
	2927	+ dwc3_writel(dwc->regs, DWC3_DCFG, reg);
	2928	+
2315	2929	/* Start with SuperSpeed Default */
2316	2930	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2317	2931
2318	2932	dep = dwc->eps[0];
	2933	+ dep->flags = 0;
2319	2934	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2320	2935	if (ret) {
2321	2936	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
..	..	@@ -2323,6 +2938,7 @@
2323	2938	}
2324	2939
2325	2940	dep = dwc->eps[1];
	2941	+ dep->flags = 0;
2326	2942	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2327	2943	if (ret) {
2328	2944	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
..	..	@@ -2331,6 +2947,7 @@
2331	2947
2332	2948	/* begin to receive SETUP packets */
2333	2949	dwc->ep0state = EP0_SETUP_PHASE;
	2950	+ dwc->ep0_bounced = false;
2334	2951	dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2335	2952	dwc->delayed_status = false;
2336	2953	dwc3_ep0_out_start(dwc);
..	..	@@ -2366,17 +2983,13 @@
2366	2983	spin_lock_irqsave(&dwc->lock, flags);
2367	2984	if (dwc->gadget_driver) {
2368	2985	dev_err(dwc->dev, "%s is already bound to %s\n",
2369		- dwc->gadget.name,
	2986	+ dwc->gadget->name,
2370	2987	dwc->gadget_driver->driver.name);
2371	2988	ret = -EBUSY;
2372	2989	goto err1;
2373	2990	}
2374	2991
2375	2992	dwc->gadget_driver = driver;
2376		-
2377		- if (pm_runtime_active(dwc->dev))
2378		- __dwc3_gadget_start(dwc);
2379		-
2380	2993	spin_unlock_irqrestore(&dwc->lock, flags);
2381	2994
2382	2995	return 0;
..	..	@@ -2402,28 +3015,59 @@
2402	3015	unsigned long flags;
2403	3016
2404	3017	spin_lock_irqsave(&dwc->lock, flags);
2405		-
2406	3018	if (!dwc->gadget_driver) {
2407	3019	spin_unlock_irqrestore(&dwc->lock, flags);
2408	3020	dev_warn(dwc->dev, "%s is already stopped\n",
2409		- dwc->gadget.name);
2410		- goto out0;
	3021	+ dwc->gadget->name);
	3022	+ goto out;
2411	3023	}
2412		-
2413		- if (pm_runtime_suspended(dwc->dev))
2414		- goto out1;
2415		-
2416		- __dwc3_gadget_stop(dwc);
2417		-
2418		-out1:
2419		- dwc->fifo_resize_status = false;
2420	3024	dwc->gadget_driver = NULL;
	3025	+ dwc->max_cfg_eps = 0;
2421	3026	spin_unlock_irqrestore(&dwc->lock, flags);
2422	3027
2423	3028	free_irq(dwc->irq_gadget, dwc->ev_buf);
2424	3029
2425		-out0:
	3030	+out:
2426	3031	return 0;
	3032	+}
	3033	+
	3034	+static void dwc3_gadget_config_params(struct usb_gadget *g,
	3035	+ struct usb_dcd_config_params *params)
	3036	+{
	3037	+ struct dwc3 *dwc = gadget_to_dwc(g);
	3038	+
	3039	+ params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
	3040	+ params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
	3041	+
	3042	+ /* Recommended BESL */
	3043	+ if (!dwc->dis_enblslpm_quirk) {
	3044	+ /*
	3045	+ * If the recommended BESL baseline is 0 or if the BESL deep is
	3046	+ * less than 2, Microsoft's Windows 10 host usb stack will issue
	3047	+ * a usb reset immediately after it receives the extended BOS
	3048	+ * descriptor and the enumeration will fail. To maintain
	3049	+ * compatibility with the Windows' usb stack, let's set the
	3050	+ * recommended BESL baseline to 1 and clamp the BESL deep to be
	3051	+ * within 2 to 15.
	3052	+ */
	3053	+ params->besl_baseline = 1;
	3054	+ if (dwc->is_utmi_l1_suspend)
	3055	+ params->besl_deep =
	3056	+ clamp_t(u8, dwc->hird_threshold, 2, 15);
	3057	+ }
	3058	+
	3059	+ /* U1 Device exit Latency */
	3060	+ if (dwc->dis_u1_entry_quirk)
	3061	+ params->bU1devExitLat = 0;
	3062	+ else
	3063	+ params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
	3064	+
	3065	+ /* U2 Device exit Latency */
	3066	+ if (dwc->dis_u2_entry_quirk)
	3067	+ params->bU2DevExitLat = 0;
	3068	+ else
	3069	+ params->bU2DevExitLat =
	3070	+ cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
2427	3071	}
2428	3072
2429	3073	static void dwc3_gadget_set_speed(struct usb_gadget *g,
..	..	@@ -2431,67 +3075,94 @@
2431	3075	{
2432	3076	struct dwc3 *dwc = gadget_to_dwc(g);
2433	3077	unsigned long flags;
2434		- u32 reg;
2435		-
2436		- /*
2437		- * To prevent Android 10 from trying to call UDC and failed constantly
2438		- * while dwc3 is suspended, we let the UDC node always exist.
2439		- * If not return here, it may cause crashes.
2440		- */
2441		- if (pm_runtime_suspended(dwc->dev))
2442		- return;
2443	3078
2444	3079	spin_lock_irqsave(&dwc->lock, flags);
2445		- reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2446		- reg &= ~(DWC3_DCFG_SPEED_MASK);
	3080	+ dwc->gadget_max_speed = speed;
	3081	+ spin_unlock_irqrestore(&dwc->lock, flags);
	3082	+}
2447	3083
2448		- /*
2449		- * WORKAROUND: DWC3 revision < 2.20a have an issue
2450		- * which would cause metastability state on Run/Stop
2451		- * bit if we try to force the IP to USB2-only mode.
2452		- *
2453		- * Because of that, we cannot configure the IP to any
2454		- * speed other than the SuperSpeed
2455		- *
2456		- * Refers to:
2457		- *
2458		- * STAR#9000525659: Clock Domain Crossing on DCTL in
2459		- * USB 2.0 Mode
2460		- */
2461		- if (dwc->revision < DWC3_REVISION_220A &&
2462		- !dwc->dis_metastability_quirk) {
2463		- reg \|= DWC3_DCFG_SUPERSPEED;
2464		- } else {
2465		- switch (speed) {
2466		- case USB_SPEED_LOW:
2467		- reg \|= DWC3_DCFG_LOWSPEED;
2468		- break;
2469		- case USB_SPEED_FULL:
2470		- reg \|= DWC3_DCFG_FULLSPEED;
2471		- break;
2472		- case USB_SPEED_HIGH:
2473		- reg \|= DWC3_DCFG_HIGHSPEED;
2474		- break;
2475		- case USB_SPEED_SUPER:
2476		- reg \|= DWC3_DCFG_SUPERSPEED;
2477		- break;
2478		- case USB_SPEED_SUPER_PLUS:
2479		- if (dwc3_is_usb31(dwc))
2480		- reg \|= DWC3_DCFG_SUPERSPEED_PLUS;
2481		- else
2482		- reg \|= DWC3_DCFG_SUPERSPEED;
2483		- break;
2484		- default:
2485		- dev_err(dwc->dev, "invalid speed (%d)\n", speed);
	3084	+static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
	3085	+ enum usb_ssp_rate rate)
	3086	+{
	3087	+ struct dwc3 *dwc = gadget_to_dwc(g);
	3088	+ unsigned long flags;
2486	3089
2487		- if (dwc->revision & DWC3_REVISION_IS_DWC31)
2488		- reg \|= DWC3_DCFG_SUPERSPEED_PLUS;
2489		- else
2490		- reg \|= DWC3_DCFG_SUPERSPEED;
2491		- }
	3090	+ spin_lock_irqsave(&dwc->lock, flags);
	3091	+ dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
	3092	+ dwc->gadget_ssp_rate = rate;
	3093	+ spin_unlock_irqrestore(&dwc->lock, flags);
	3094	+}
	3095	+
	3096	+static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
	3097	+{
	3098	+ struct dwc3 *dwc = gadget_to_dwc(g);
	3099	+ union power_supply_propval val = {0};
	3100	+ int ret;
	3101	+
	3102	+ if (dwc->usb2_phy)
	3103	+ return usb_phy_set_power(dwc->usb2_phy, mA);
	3104	+
	3105	+ if (!dwc->usb_psy)
	3106	+ return -EOPNOTSUPP;
	3107	+
	3108	+ val.intval = 1000 * mA;
	3109	+ ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
	3110	+
	3111	+ return ret;
	3112	+}
	3113	+
	3114	+/**
	3115	+ * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
	3116	+ * @g: pointer to the USB gadget
	3117	+ *
	3118	+ * Used to record the maximum number of endpoints being used in a USB composite
	3119	+ * device. (across all configurations) This is to be used in the calculation
	3120	+ * of the TXFIFO sizes when resizing internal memory for individual endpoints.
	3121	+ * It will help ensured that the resizing logic reserves enough space for at
	3122	+ * least one max packet.
	3123	+ */
	3124	+static int dwc3_gadget_check_config(struct usb_gadget *g)
	3125	+{
	3126	+ struct dwc3 *dwc = gadget_to_dwc(g);
	3127	+ struct usb_ep *ep;
	3128	+ int fifo_size = 0;
	3129	+ int ram1_depth;
	3130	+ int ep_num = 0;
	3131	+
	3132	+ if (!dwc->do_fifo_resize)
	3133	+ return 0;
	3134	+
	3135	+ list_for_each_entry(ep, &g->ep_list, ep_list) {
	3136	+ /* Only interested in the IN endpoints */
	3137	+ if (ep->claimed && (ep->address & USB_DIR_IN))
	3138	+ ep_num++;
2492	3139	}
2493		- dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2494	3140
	3141	+ if (ep_num <= dwc->max_cfg_eps)
	3142	+ return 0;
	3143	+
	3144	+ /* Update the max number of eps in the composition */
	3145	+ dwc->max_cfg_eps = ep_num;
	3146	+
	3147	+ fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);
	3148	+ /* Based on the equation, increment by one for every ep */
	3149	+ fifo_size += dwc->max_cfg_eps;
	3150	+
	3151	+ /* Check if we can fit a single fifo per endpoint */
	3152	+ ram1_depth = dwc3_gadget_get_tx_fifos_size(dwc);
	3153	+ if (fifo_size > ram1_depth)
	3154	+ return -ENOMEM;
	3155	+
	3156	+ return 0;
	3157	+}
	3158	+
	3159	+static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
	3160	+{
	3161	+ struct dwc3 *dwc = gadget_to_dwc(g);
	3162	+ unsigned long flags;
	3163	+
	3164	+ spin_lock_irqsave(&dwc->lock, flags);
	3165	+ dwc->async_callbacks = enable;
2495	3166	spin_unlock_irqrestore(&dwc->lock, flags);
2496	3167	}
2497	3168
..	..	@@ -2503,6 +3174,11 @@
2503	3174	.udc_start = dwc3_gadget_start,
2504	3175	.udc_stop = dwc3_gadget_stop,
2505	3176	.udc_set_speed = dwc3_gadget_set_speed,
	3177	+ .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate,
	3178	+ .get_config_params = dwc3_gadget_config_params,
	3179	+ .vbus_draw = dwc3_gadget_vbus_draw,
	3180	+ .check_config = dwc3_gadget_check_config,
	3181	+ .udc_async_callbacks = dwc3_gadget_async_callbacks,
2506	3182	};
2507	3183
2508	3184	/* -------------------------------------------------------------------------- */
..	..	@@ -2515,7 +3191,7 @@
2515	3191	dep->endpoint.maxburst = 1;
2516	3192	dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2517	3193	if (!dep->direction)
2518		- dwc->gadget.ep0 = &dep->endpoint;
	3194	+ dwc->gadget->ep0 = &dep->endpoint;
2519	3195
2520	3196	dep->endpoint.caps.type_control = true;
2521	3197
..	..	@@ -2525,48 +3201,62 @@
2525	3201	static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2526	3202	{
2527	3203	struct dwc3 *dwc = dep->dwc;
2528		- int mdwidth;
	3204	+ u32 mdwidth;
2529	3205	int size;
	3206	+ int maxpacket;
2530	3207
2531		- mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
	3208	+ mdwidth = dwc3_mdwidth(dwc);
	3209	+
2532	3210	/* MDWIDTH is represented in bits, we need it in bytes */
2533	3211	mdwidth /= 8;
2534	3212
2535	3213	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2536		- if (dwc3_is_usb31(dwc))
2537		- size = DWC31_GTXFIFOSIZ_TXFDEF(size);
	3214	+ if (DWC3_IP_IS(DWC3))
	3215	+ size = DWC3_GTXFIFOSIZ_TXFDEP(size);
2538	3216	else
2539		- size = DWC3_GTXFIFOSIZ_TXFDEF(size);
2540		-
2541		- /* FIFO Depth is in MDWDITH bytes. Multiply */
2542		- size *= mdwidth;
	3217	+ size = DWC31_GTXFIFOSIZ_TXFDEP(size);
2543	3218
2544	3219	/*
2545		- * To meet performance requirement, a minimum TxFIFO size of 3x
2546		- * MaxPacketSize is recommended for endpoints that support burst and a
2547		- * minimum TxFIFO size of 2x MaxPacketSize for endpoints that don't
2548		- * support burst. Use those numbers and we can calculate the max packet
2549		- * limit as below.
	3220	+ * maxpacket size is determined as part of the following, after assuming
	3221	+ * a mult value of one maxpacket:
	3222	+ * DWC3 revision 280A and prior:
	3223	+ * fifo_size = mult * (max_packet / mdwidth) + 1;
	3224	+ * maxpacket = mdwidth * (fifo_size - 1);
	3225	+ *
	3226	+ * DWC3 revision 290A and onwards:
	3227	+ * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
	3228	+ * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
2550	3229	*/
2551		- if (dwc->maximum_speed >= USB_SPEED_SUPER)
2552		- size /= 3;
	3230	+ if (DWC3_VER_IS_PRIOR(DWC3, 290A))
	3231	+ maxpacket = mdwidth * (size - 1);
2553	3232	else
2554		- size /= 2;
	3233	+ maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
2555	3234
	3235	+
	3236	+ /*
	3237	+ * To meet performance requirement, a minimum TxFIFO size of 2x
	3238	+ * MaxPacketSize is recommended for endpoints that support for
	3239	+ * Rockchip platform with UVC function.
	3240	+ */
	3241	+ if (IS_REACHABLE(CONFIG_ARCH_ROCKCHIP) &&
	3242	+ (dwc->maximum_speed >= USB_SPEED_HIGH))
	3243	+ maxpacket /= 2;
	3244	+
	3245	+ /* Functionally, space for one max packet is sufficient */
	3246	+ size = min_t(int, maxpacket, 1024);
2556	3247	/*
2557	3248	* If enable tx fifos resize, set each in ep maxpacket
2558	3249	* to 1024, it can avoid being dependent on the default
2559	3250	* fifo size, and more flexible use of endpoints.
2560	3251	*/
2561		- if (dwc->needs_fifo_resize)
	3252	+ if (dwc->do_fifo_resize)
2562	3253	size = 1024;
2563		-
2564	3254	usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2565	3255
2566		- dep->endpoint.max_streams = 15;
	3256	+ dep->endpoint.max_streams = 16;
2567	3257	dep->endpoint.ops = &dwc3_gadget_ep_ops;
2568	3258	list_add_tail(&dep->endpoint.ep_list,
2569		- &dwc->gadget.ep_list);
	3259	+ &dwc->gadget->ep_list);
2570	3260	dep->endpoint.caps.type_iso = true;
2571	3261	dep->endpoint.caps.type_bulk = true;
2572	3262	dep->endpoint.caps.type_int = true;
..	..	@@ -2577,20 +3267,20 @@
2577	3267	static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2578	3268	{
2579	3269	struct dwc3 *dwc = dep->dwc;
2580		- int mdwidth;
	3270	+ u32 mdwidth;
2581	3271	int size;
2582	3272
2583		- mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
	3273	+ mdwidth = dwc3_mdwidth(dwc);
2584	3274
2585	3275	/* MDWIDTH is represented in bits, convert to bytes */
2586	3276	mdwidth /= 8;
2587	3277
2588	3278	/* All OUT endpoints share a single RxFIFO space */
2589	3279	size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
2590		- if (dwc3_is_usb31(dwc))
2591		- size = DWC31_GRXFIFOSIZ_RXFDEP(size);
2592		- else
	3280	+ if (DWC3_IP_IS(DWC3))
2593	3281	size = DWC3_GRXFIFOSIZ_RXFDEP(size);
	3282	+ else
	3283	+ size = DWC31_GRXFIFOSIZ_RXFDEP(size);
2594	3284
2595	3285	/* FIFO depth is in MDWDITH bytes */
2596	3286	size *= mdwidth;
..	..	@@ -2610,10 +3300,10 @@
2610	3300	size /= 3;
2611	3301
2612	3302	usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2613		- dep->endpoint.max_streams = 15;
	3303	+ dep->endpoint.max_streams = 16;
2614	3304	dep->endpoint.ops = &dwc3_gadget_ep_ops;
2615	3305	list_add_tail(&dep->endpoint.ep_list,
2616		- &dwc->gadget.ep_list);
	3306	+ &dwc->gadget->ep_list);
2617	3307	dep->endpoint.caps.type_iso = true;
2618	3308	dep->endpoint.caps.type_bulk = true;
2619	3309	dep->endpoint.caps.type_int = true;
..	..	@@ -2627,20 +3317,20 @@
2627	3317	bool direction = epnum & 1;
2628	3318	int ret;
2629	3319	u8 num = epnum >> 1;
2630		- u8 num_in_eps, num_out_eps;
2631		-
2632		- num_in_eps = DWC3_NUM_IN_EPS(&dwc->hwparams);
2633		- num_out_eps = dwc->num_eps - num_in_eps;
	3320	+ u8 num_in_eps, num_out_eps, min_eps;
2634	3321
2635	3322	dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2636	3323	if (!dep)
2637	3324	return -ENOMEM;
2638	3325
	3326	+ num_in_eps = DWC3_NUM_IN_EPS(&dwc->hwparams);
	3327	+ num_out_eps = dwc->num_eps - num_in_eps;
	3328	+ min_eps = min_t(u8, num_in_eps, num_out_eps);
	3329	+
2639	3330	/* reconfig direction and num if num_out_eps != num_in_eps */
2640		- if ((!direction && ((epnum >> 1) + 1) > num_out_eps) \|\|
2641		- (direction && ((epnum >> 1) + 1) > num_in_eps)) {
2642		- direction = !direction;
2643		- num = num + (epnum & 1);
	3331	+ if (num + 1 > min_eps && num_in_eps != num_out_eps) {
	3332	+ num = epnum - min_eps;
	3333	+ direction = num + 1 > num_out_eps ? 1 : 0;
2644	3334	}
2645	3335
2646	3336	dep->dwc = dwc;
..	..	@@ -2659,12 +3349,7 @@
2659	3349	if (!(dep->number > 1)) {
2660	3350	dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2661	3351	dep->endpoint.comp_desc = NULL;
2662		-#ifdef CONFIG_ARCH_ROCKCHIP
2663		- dep->endpoint.transfer_type = USB_ENDPOINT_XFER_CONTROL;
2664		-#endif
2665	3352	}
2666		-
2667		- spin_lock_init(&dep->lock);
2668	3353
2669	3354	if (num == 0)
2670	3355	ret = dwc3_gadget_init_control_endpoint(dep);
..	..	@@ -2692,7 +3377,7 @@
2692	3377	{
2693	3378	u8 epnum;
2694	3379
2695		- INIT_LIST_HEAD(&dwc->gadget.ep_list);
	3380	+ INIT_LIST_HEAD(&dwc->gadget->ep_list);
2696	3381
2697	3382	for (epnum = 0; epnum < total; epnum++) {
2698	3383	int ret;
..	..	@@ -2773,12 +3458,12 @@
2773	3458	}
2774	3459
2775	3460	/*
2776		- * If we're dealing with unaligned size OUT transfer, we will be left
2777		- * with one TRB pending in the ring. We need to manually clear HWO bit
2778		- * from that TRB.
	3461	+ * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
	3462	+ * this TRB points to the bounce buffer address, it's a MPS alignment
	3463	+ * TRB. Don't add it to req->remaining calculation.
2779	3464	*/
2780		-
2781		- if (req->needs_extra_trb && !(trb->ctrl & DWC3_TRB_CTRL_CHN)) {
	3465	+ if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
	3466	+ trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
2782	3467	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2783	3468	return 1;
2784	3469	}
..	..	@@ -2790,6 +3475,10 @@
2790	3475	return 1;
2791	3476
2792	3477	if (event->status & DEPEVT_STATUS_SHORT && !chain)
	3478	+ return 1;
	3479	+
	3480	+ if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&
	3481	+ DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)
2793	3482	return 1;
2794	3483
2795	3484	if ((trb->ctrl & DWC3_TRB_CTRL_IOC) \|\|
..	..	@@ -2845,6 +3534,7 @@
2845	3534	struct dwc3_request *req, int status)
2846	3535	{
2847	3536	struct dwc3 *dwc = dep->dwc;
	3537	+ int request_status;
2848	3538	int ret;
2849	3539
2850	3540	if (req->request.num_mapped_sgs)
..	..	@@ -2856,35 +3546,26 @@
2856	3546
2857	3547	req->request.actual = req->request.length - req->remaining;
2858	3548
2859		- if (!dwc3_gadget_ep_request_completed(req) &&
2860		- !usb_endpoint_xfer_isoc(dep->endpoint.desc))
	3549	+ if (!dwc3_gadget_ep_request_completed(req))
2861	3550	goto out;
2862	3551
2863	3552	if (req->needs_extra_trb) {
2864		- unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
2865		-
2866	3553	ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2867	3554	status);
2868		-
2869		- /* Reclaim MPS padding TRB for ZLP */
2870		- if (!req->direction && req->request.zero && req->request.length &&
2871		- !usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2872		- (IS_ALIGNED(req->request.length, maxp)))
2873		- ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, status);
2874		-
2875	3555	req->needs_extra_trb = false;
2876	3556	}
2877	3557
2878		- if (event->status & DEPEVT_STATUS_MISSED_ISOC &&
2879		- usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2880		- /*
2881		- * unmap isoc request and move the request
2882		- * to the pending list to wait for kicking
2883		- * transfer again.
2884		- */
	3558	+ /*
	3559	+ * If MISS ISOC happens, we need to move the req from started_list
	3560	+ * to cancelled_list, then unmap the req and clear the HWO of trb.
	3561	+ * Later in the dwc3_gadget_endpoint_trbs_complete(), it will move
	3562	+ * the req from the cancelled_list to the pending_list, and restart
	3563	+ * the req for isoc transfer.
	3564	+ */
	3565	+ if (status == -EXDEV && usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2885	3566	req->remaining = 0;
2886	3567	req->needs_extra_trb = false;
2887		- dwc3_gadget_move_cancelled_request(req);
	3568	+ dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
2888	3569	if (req->trb) {
2889	3570	usb_gadget_unmap_request_by_dev(dwc->sysdev,
2890	3571	&req->request,
..	..	@@ -2893,11 +3574,38 @@
2893	3574	req->trb = NULL;
2894	3575	}
2895	3576	ret = 0;
2896		-
2897	3577	goto out;
2898	3578	}
2899	3579
2900		- dwc3_gadget_giveback(dep, req, status);
	3580	+ /*
	3581	+ * The event status only reflects the status of the TRB with IOC set.
	3582	+ * For the requests that don't set interrupt on completion, the driver
	3583	+ * needs to check and return the status of the completed TRBs associated
	3584	+ * with the request. Use the status of the last TRB of the request.
	3585	+ */
	3586	+ if (req->request.no_interrupt) {
	3587	+ struct dwc3_trb *trb;
	3588	+
	3589	+ trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
	3590	+ switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
	3591	+ case DWC3_TRBSTS_MISSED_ISOC:
	3592	+ /* Isoc endpoint only */
	3593	+ request_status = -EXDEV;
	3594	+ break;
	3595	+ case DWC3_TRB_STS_XFER_IN_PROG:
	3596	+ /* Applicable when End Transfer with ForceRM=0 */
	3597	+ case DWC3_TRBSTS_SETUP_PENDING:
	3598	+ /* Control endpoint only */
	3599	+ case DWC3_TRBSTS_OK:
	3600	+ default:
	3601	+ request_status = 0;
	3602	+ break;
	3603	+ }
	3604	+ } else {
	3605	+ request_status = status;
	3606	+ }
	3607	+
	3608	+ dwc3_gadget_giveback(dep, req, request_status);
2901	3609
2902	3610	out:
2903	3611	return ret;
..	..	@@ -2907,14 +3615,20 @@
2907	3615	const struct dwc3_event_depevt *event, int status)
2908	3616	{
2909	3617	struct dwc3_request *req;
2910		- struct dwc3_request *tmp;
2911	3618
2912		- list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
	3619	+ while (!list_empty(&dep->started_list)) {
2913	3620	int ret;
2914	3621
	3622	+ req = next_request(&dep->started_list);
2915	3623	ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2916	3624	req, status);
2917	3625	if (ret)
	3626	+ break;
	3627	+ /*
	3628	+ * The endpoint is disabled, let the dwc3_remove_requests()
	3629	+ * handle the cleanup.
	3630	+ */
	3631	+ if (!dep->endpoint.desc)
2918	3632	break;
2919	3633	}
2920	3634	}
..	..	@@ -2922,6 +3636,11 @@
2922	3636	static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
2923	3637	{
2924	3638	struct dwc3_request *req;
	3639	+ struct dwc3 *dwc = dep->dwc;
	3640	+
	3641	+ if (!dep->endpoint.desc \|\| !dwc->pullups_connected \|\|
	3642	+ !dwc->connected)
	3643	+ return false;
2925	3644
2926	3645	if (!list_empty(&dep->pending_list))
2927	3646	return true;
..	..	@@ -2943,49 +3662,58 @@
2943	3662	dep->frame_number = event->parameters;
2944	3663	}
2945	3664
2946		-static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2947		- const struct dwc3_event_depevt *event)
	3665	+static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
	3666	+ const struct dwc3_event_depevt *event, int status)
2948	3667	{
2949	3668	struct dwc3 *dwc = dep->dwc;
2950		- unsigned status = 0;
2951		- struct dwc3_request *req;
2952		- struct dwc3_request *tmp;
2953		-
2954		- dwc3_gadget_endpoint_frame_from_event(dep, event);
2955		-
2956		- if (event->status & DEPEVT_STATUS_BUSERR)
2957		- status = -ECONNRESET;
2958		-
2959		- if (event->status & DEPEVT_STATUS_MISSED_ISOC)
2960		- status = -EXDEV;
	3669	+ struct dwc3_request req, tmp;
	3670	+ bool no_started_trb = true;
2961	3671
2962	3672	dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2963	3673
2964		- if (event->status & DEPEVT_STATUS_MISSED_ISOC &&
	3674	+ if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
	3675	+ goto out;
	3676	+
	3677	+ if (!dep->endpoint.desc)
	3678	+ return no_started_trb;
	3679	+
	3680	+ /*
	3681	+ * If MISS ISOC happens, we need to do the following three steps
	3682	+ * to restart the reqs in the cancelled_list and pending_list
	3683	+ * in order.
	3684	+ * Step1. Move all the reqs from pending_list to the tail of
	3685	+ * cancelled_list.
	3686	+ * Step2. Move all the reqs from cancelled_list to the tail
	3687	+ * of pending_list.
	3688	+ * Step3. Stop and restart an isoc transfer.
	3689	+ */
	3690	+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && status == -EXDEV &&
2965	3691	!list_empty(&dep->cancelled_list) &&
2966	3692	!list_empty(&dep->pending_list)) {
2967	3693	list_for_each_entry_safe(req, tmp, &dep->pending_list, list)
2968		- dwc3_gadget_move_cancelled_request(req);
	3694	+ dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
2969	3695	}
2970	3696
2971		- if (event->status & DEPEVT_STATUS_MISSED_ISOC &&
	3697	+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && status == -EXDEV &&
2972	3698	!list_empty(&dep->cancelled_list)) {
2973	3699	list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list)
2974	3700	dwc3_gadget_move_queued_request(req);
2975	3701	}
2976	3702
2977		- if (event->status & DEPEVT_STATUS_MISSED_ISOC &&
2978		- list_empty(&dep->started_list))
	3703	+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
	3704	+ list_empty(&dep->started_list) &&
	3705	+ (list_empty(&dep->pending_list) \|\| status == -EXDEV))
2979	3706	dwc3_stop_active_transfer(dep, true, true);
2980		- else if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2981		- dwc3_gadget_ep_should_continue(dep))
2982		- __dwc3_gadget_kick_transfer(dep);
	3707	+ else if (dwc3_gadget_ep_should_continue(dep))
	3708	+ if (__dwc3_gadget_kick_transfer(dep) == 0)
	3709	+ no_started_trb = false;
2983	3710
	3711	+out:
2984	3712	/*
2985	3713	* WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2986	3714	* See dwc3_gadget_linksts_change_interrupt() for 1st half.
2987	3715	*/
2988		- if (dwc->revision < DWC3_REVISION_183A) {
	3716	+ if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
2989	3717	u32 reg;
2990	3718	int i;
2991	3719
..	..	@@ -2996,7 +3724,7 @@
2996	3724	continue;
2997	3725
2998	3726	if (!list_empty(&dep->started_list))
2999		- return;
	3727	+ return no_started_trb;
3000	3728	}
3001	3729
3002	3730	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
..	..	@@ -3005,13 +3733,171 @@
3005	3733
3006	3734	dwc->u1u2 = 0;
3007	3735	}
	3736	+
	3737	+ return no_started_trb;
	3738	+}
	3739	+
	3740	+static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
	3741	+ const struct dwc3_event_depevt *event)
	3742	+{
	3743	+ int status = 0;
	3744	+
	3745	+ if (!dep->endpoint.desc)
	3746	+ return;
	3747	+
	3748	+ if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
	3749	+ dwc3_gadget_endpoint_frame_from_event(dep, event);
	3750	+
	3751	+ if (event->status & DEPEVT_STATUS_BUSERR)
	3752	+ status = -ECONNRESET;
	3753	+
	3754	+ if (event->status & DEPEVT_STATUS_MISSED_ISOC)
	3755	+ status = -EXDEV;
	3756	+
	3757	+ dwc3_gadget_endpoint_trbs_complete(dep, event, status);
	3758	+}
	3759	+
	3760	+static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
	3761	+ const struct dwc3_event_depevt *event)
	3762	+{
	3763	+ int status = 0;
	3764	+
	3765	+ dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
	3766	+
	3767	+ if (event->status & DEPEVT_STATUS_BUSERR)
	3768	+ status = -ECONNRESET;
	3769	+
	3770	+ if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
	3771	+ dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3008	3772	}
3009	3773
3010	3774	static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3011	3775	const struct dwc3_event_depevt *event)
3012	3776	{
3013	3777	dwc3_gadget_endpoint_frame_from_event(dep, event);
	3778	+
	3779	+ /*
	3780	+ * The XferNotReady event is generated only once before the endpoint
	3781	+ * starts. It will be generated again when END_TRANSFER command is
	3782	+ * issued. For some controller versions, the XferNotReady event may be
	3783	+ * generated while the END_TRANSFER command is still in process. Ignore
	3784	+ * it and wait for the next XferNotReady event after the command is
	3785	+ * completed.
	3786	+ */
	3787	+ if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
	3788	+ return;
	3789	+
3014	3790	(void) __dwc3_gadget_start_isoc(dep);
	3791	+}
	3792	+
	3793	+static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
	3794	+ const struct dwc3_event_depevt *event)
	3795	+{
	3796	+ u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
	3797	+
	3798	+ if (cmd != DWC3_DEPCMD_ENDTRANSFER)
	3799	+ return;
	3800	+
	3801	+ /*
	3802	+ * The END_TRANSFER command will cause the controller to generate a
	3803	+ * NoStream Event, and it's not due to the host DP NoStream rejection.
	3804	+ * Ignore the next NoStream event.
	3805	+ */
	3806	+ if (dep->stream_capable)
	3807	+ dep->flags \|= DWC3_EP_IGNORE_NEXT_NOSTREAM;
	3808	+
	3809	+ dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
	3810	+ dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
	3811	+ dwc3_gadget_ep_cleanup_cancelled_requests(dep);
	3812	+
	3813	+ if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
	3814	+ struct dwc3 *dwc = dep->dwc;
	3815	+ struct dwc3_vendor *vdwc = container_of(dwc, struct dwc3_vendor, dwc);
	3816	+
	3817	+ dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
	3818	+ if (dwc3_send_clear_stall_ep_cmd(dep)) {
	3819	+ struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
	3820	+
	3821	+ dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
	3822	+ if (dwc->delayed_status)
	3823	+ __dwc3_gadget_ep0_set_halt(ep0, 1);
	3824	+ return;
	3825	+ }
	3826	+
	3827	+ dep->flags &= ~(DWC3_EP_STALL \| DWC3_EP_WEDGE);
	3828	+ if (vdwc->clear_stall_protocol == dep->number)
	3829	+ dwc3_ep0_send_delayed_status(dwc);
	3830	+ }
	3831	+
	3832	+ if ((dep->flags & DWC3_EP_DELAY_START) &&
	3833	+ !usb_endpoint_xfer_isoc(dep->endpoint.desc))
	3834	+ __dwc3_gadget_kick_transfer(dep);
	3835	+
	3836	+ dep->flags &= ~DWC3_EP_DELAY_START;
	3837	+}
	3838	+
	3839	+static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
	3840	+ const struct dwc3_event_depevt *event)
	3841	+{
	3842	+ struct dwc3 *dwc = dep->dwc;
	3843	+
	3844	+ if (event->status == DEPEVT_STREAMEVT_FOUND) {
	3845	+ dep->flags \|= DWC3_EP_FIRST_STREAM_PRIMED;
	3846	+ goto out;
	3847	+ }
	3848	+
	3849	+ /* Note: NoStream rejection event param value is 0 and not 0xFFFF */
	3850	+ switch (event->parameters) {
	3851	+ case DEPEVT_STREAM_PRIME:
	3852	+ /*
	3853	+ * If the host can properly transition the endpoint state from
	3854	+ * idle to prime after a NoStream rejection, there's no need to
	3855	+ * force restarting the endpoint to reinitiate the stream. To
	3856	+ * simplify the check, assume the host follows the USB spec if
	3857	+ * it primed the endpoint more than once.
	3858	+ */
	3859	+ if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
	3860	+ if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
	3861	+ dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
	3862	+ else
	3863	+ dep->flags \|= DWC3_EP_FIRST_STREAM_PRIMED;
	3864	+ }
	3865	+
	3866	+ break;
	3867	+ case DEPEVT_STREAM_NOSTREAM:
	3868	+ if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) \|\|
	3869	+ !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) \|\|
	3870	+ !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))
	3871	+ break;
	3872	+
	3873	+ /*
	3874	+ * If the host rejects a stream due to no active stream, by the
	3875	+ * USB and xHCI spec, the endpoint will be put back to idle
	3876	+ * state. When the host is ready (buffer added/updated), it will
	3877	+ * prime the endpoint to inform the usb device controller. This
	3878	+ * triggers the device controller to issue ERDY to restart the
	3879	+ * stream. However, some hosts don't follow this and keep the
	3880	+ * endpoint in the idle state. No prime will come despite host
	3881	+ * streams are updated, and the device controller will not be
	3882	+ * triggered to generate ERDY to move the next stream data. To
	3883	+ * workaround this and maintain compatibility with various
	3884	+ * hosts, force to reinitate the stream until the host is ready
	3885	+ * instead of waiting for the host to prime the endpoint.
	3886	+ */
	3887	+ if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
	3888	+ unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
	3889	+
	3890	+ dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
	3891	+ } else {
	3892	+ dep->flags \|= DWC3_EP_DELAY_START;
	3893	+ dwc3_stop_active_transfer(dep, true, true);
	3894	+ return;
	3895	+ }
	3896	+ break;
	3897	+ }
	3898	+
	3899	+out:
	3900	+ dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3015	3901	}
3016	3902
3017	3903	static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
..	..	@@ -3019,25 +3905,20 @@
3019	3905	{
3020	3906	struct dwc3_ep *dep;
3021	3907	u8 epnum = event->endpoint_number;
3022		- u8 cmd;
3023	3908
3024	3909	dep = dwc->eps[epnum];
3025	3910
3026	3911	if (!(dep->flags & DWC3_EP_ENABLED)) {
3027		- if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
	3912	+ if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3028	3913	return;
3029	3914
3030	3915	/* Handle only EPCMDCMPLT when EP disabled */
3031		- if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
	3916	+ if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&
	3917	+ !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))
3032	3918	return;
3033	3919	}
3034	3920
3035	3921	if (epnum == 0 \|\| epnum == 1) {
3036		- if (!dwc->connected &&
3037		- event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE) {
3038		- reinit_completion(&dwc->discon_done);
3039		- dwc->connected = true;
3040		- }
3041	3922	dwc3_ep0_interrupt(dwc, event);
3042	3923	return;
3043	3924	}
..	..	@@ -3050,21 +3931,14 @@
3050	3931	dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3051	3932	break;
3052	3933	case DWC3_DEPEVT_EPCMDCMPLT:
3053		- cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3054		-
3055		- if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
3056		- dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3057		- dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3058		- dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3059		- if ((dep->flags & DWC3_EP_DELAY_START) &&
3060		- !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3061		- __dwc3_gadget_kick_transfer(dep);
3062		-
3063		- dep->flags &= ~DWC3_EP_DELAY_START;
3064		- }
	3934	+ dwc3_gadget_endpoint_command_complete(dep, event);
	3935	+ break;
	3936	+ case DWC3_DEPEVT_XFERCOMPLETE:
	3937	+ dwc3_gadget_endpoint_transfer_complete(dep, event);
3065	3938	break;
3066	3939	case DWC3_DEPEVT_STREAMEVT:
3067		- case DWC3_DEPEVT_XFERCOMPLETE:
	3940	+ dwc3_gadget_endpoint_stream_event(dep, event);
	3941	+ break;
3068	3942	case DWC3_DEPEVT_RXTXFIFOEVT:
3069	3943	break;
3070	3944	}
..	..	@@ -3072,27 +3946,27 @@
3072	3946
3073	3947	static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3074	3948	{
3075		- if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
	3949	+ if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3076	3950	spin_unlock(&dwc->lock);
3077		- dwc->gadget_driver->disconnect(&dwc->gadget);
	3951	+ dwc->gadget_driver->disconnect(dwc->gadget);
3078	3952	spin_lock(&dwc->lock);
3079	3953	}
3080	3954	}
3081	3955
3082	3956	static void dwc3_suspend_gadget(struct dwc3 *dwc)
3083	3957	{
3084		- if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
	3958	+ if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3085	3959	spin_unlock(&dwc->lock);
3086		- dwc->gadget_driver->suspend(&dwc->gadget);
	3960	+ dwc->gadget_driver->suspend(dwc->gadget);
3087	3961	spin_lock(&dwc->lock);
3088	3962	}
3089	3963	}
3090	3964
3091	3965	static void dwc3_resume_gadget(struct dwc3 *dwc)
3092	3966	{
3093		- if (dwc->gadget_driver && dwc->gadget_driver->resume) {
	3967	+ if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3094	3968	spin_unlock(&dwc->lock);
3095		- dwc->gadget_driver->resume(&dwc->gadget);
	3969	+ dwc->gadget_driver->resume(dwc->gadget);
3096	3970	spin_lock(&dwc->lock);
3097	3971	}
3098	3972	}
..	..	@@ -3102,23 +3976,45 @@
3102	3976	if (!dwc->gadget_driver)
3103	3977	return;
3104	3978
3105		- if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
	3979	+ if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3106	3980	spin_unlock(&dwc->lock);
3107		- usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
	3981	+ usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3108	3982	spin_lock(&dwc->lock);
3109	3983	}
3110	3984	}
3111	3985
3112		-static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
	3986	+void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3113	3987	bool interrupt)
3114	3988	{
3115		- struct dwc3_gadget_ep_cmd_params params;
3116		- u32 cmd;
3117		- int ret;
	3989	+ struct dwc3 *dwc = dep->dwc;
	3990	+
	3991	+ /*
	3992	+ * Only issue End Transfer command to the control endpoint of a started
	3993	+ * Data Phase. Typically we should only do so in error cases such as
	3994	+ * invalid/unexpected direction as described in the control transfer
	3995	+ * flow of the programming guide.
	3996	+ */
	3997	+ if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
	3998	+ return;
	3999	+
	4000	+ if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))
	4001	+ return;
3118	4002
3119	4003	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) \|\|
3120	4004	(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3121	4005	return;
	4006	+
	4007	+ /*
	4008	+ * If a Setup packet is received but yet to DMA out, the controller will
	4009	+ * not process the End Transfer command of any endpoint. Polling of its
	4010	+ * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
	4011	+ * timeout. Delay issuing the End Transfer command until the Setup TRB is
	4012	+ * prepared.
	4013	+ */
	4014	+ if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
	4015	+ dep->flags \|= DWC3_EP_DELAY_STOP;
	4016	+ return;
	4017	+ }
3122	4018
3123	4019	/*
3124	4020	* NOTICE: We are violating what the Databook says about the
..	..	@@ -3144,23 +4040,16 @@
3144	4040	* enabled, the EndTransfer command will have completed upon
3145	4041	* returning from this function.
3146	4042	*
3147		- * This mode is NOT available on the DWC_usb31 IP.
	4043	+ * This mode is NOT available on the DWC_usb31 IP. In this
	4044	+ * case, if the IOC bit is not set, then delay by 1ms
	4045	+ * after issuing the EndTransfer command. This allows for the
	4046	+ * controller to handle the command completely before DWC3
	4047	+ * remove requests attempts to unmap USB request buffers.
3148	4048	*/
3149	4049
3150		- cmd = DWC3_DEPCMD_ENDTRANSFER;
3151		- cmd \|= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
3152		- cmd \|= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
3153		- cmd \|= DWC3_DEPCMD_PARAM(dep->resource_index);
3154		- memset(&params, 0, sizeof(params));
3155		- ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
3156		- WARN_ON_ONCE(ret);
3157		- dep->resource_index = 0;
3158		-
3159		- if (!interrupt)
3160		- dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3161		- else
3162		- dep->flags \|= DWC3_EP_END_TRANSFER_PENDING;
	4050	+ __dwc3_stop_active_transfer(dep, force, interrupt);
3163	4051	}
	4052	+EXPORT_SYMBOL_GPL(dwc3_stop_active_transfer);
3164	4053
3165	4054	static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3166	4055	{
..	..	@@ -3188,30 +4077,36 @@
3188	4077	{
3189	4078	int reg;
3190	4079
	4080	+ dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
	4081	+
3191	4082	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3192	4083	reg &= ~DWC3_DCTL_INITU1ENA;
3193		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3194		-
3195	4084	reg &= ~DWC3_DCTL_INITU2ENA;
3196		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	4085	+ dwc3_gadget_dctl_write_safe(dwc, reg);
	4086	+
	4087	+ dwc->connected = false;
3197	4088
3198	4089	dwc3_disconnect_gadget(dwc);
3199	4090
3200		- dwc->gadget.speed = USB_SPEED_UNKNOWN;
	4091	+ dwc->gadget->speed = USB_SPEED_UNKNOWN;
3201	4092	dwc->setup_packet_pending = false;
3202		- usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
	4093	+ usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3203	4094
3204		- dwc->connected = false;
3205		- complete(&dwc->discon_done);
	4095	+ if (dwc->ep0state != EP0_SETUP_PHASE) {
	4096	+ unsigned int dir;
	4097	+
	4098	+ dir = !!dwc->ep0_expect_in;
	4099	+ if (dwc->ep0state == EP0_DATA_PHASE)
	4100	+ dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
	4101	+ else
	4102	+ dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
	4103	+ dwc3_ep0_stall_and_restart(dwc);
	4104	+ }
3206	4105	}
3207	4106
3208	4107	static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3209	4108	{
3210	4109	u32 reg;
3211		-
3212		- if (of_device_is_compatible(dwc->dev->parent->of_node,
3213		- "rockchip,rk3399-dwc3"))
3214		- phy_calibrate(dwc->usb2_generic_phy);
3215	4110
3216	4111	/*
3217	4112	* Ideally, dwc3_reset_gadget() would trigger the function
..	..	@@ -3248,16 +4143,45 @@
3248	4143	* STAR#9000466709: RTL: Device : Disconnect event not
3249	4144	* generated if setup packet pending in FIFO
3250	4145	*/
3251		- if (dwc->revision < DWC3_REVISION_188A) {
	4146	+ if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
3252	4147	if (dwc->setup_packet_pending)
3253	4148	dwc3_gadget_disconnect_interrupt(dwc);
3254	4149	}
3255	4150
3256	4151	dwc3_reset_gadget(dwc);
3257	4152
	4153	+ /*
	4154	+ * From SNPS databook section 8.1.2, the EP0 should be in setup
	4155	+ * phase. So ensure that EP0 is in setup phase by issuing a stall
	4156	+ * and restart if EP0 is not in setup phase.
	4157	+ */
	4158	+ if (dwc->ep0state != EP0_SETUP_PHASE) {
	4159	+ unsigned int dir;
	4160	+
	4161	+ dir = !!dwc->ep0_expect_in;
	4162	+ if (dwc->ep0state == EP0_DATA_PHASE)
	4163	+ dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
	4164	+ else
	4165	+ dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
	4166	+
	4167	+ dwc->eps[0]->trb_enqueue = 0;
	4168	+ dwc->eps[1]->trb_enqueue = 0;
	4169	+
	4170	+ dwc3_ep0_stall_and_restart(dwc);
	4171	+ }
	4172	+
	4173	+ /*
	4174	+ * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
	4175	+ * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
	4176	+ * needs to ensure that it sends "a DEPENDXFER command for any active
	4177	+ * transfers."
	4178	+ */
	4179	+ dwc3_stop_active_transfers(dwc);
	4180	+ dwc->connected = true;
	4181	+
3258	4182	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3259	4183	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
3260		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	4184	+ dwc3_gadget_dctl_write_safe(dwc, reg);
3261	4185	dwc->test_mode = false;
3262	4186	dwc3_clear_stall_all_ep(dwc);
3263	4187
..	..	@@ -3272,11 +4196,21 @@
3272	4196	struct dwc3_ep *dep;
3273	4197	int ret;
3274	4198	u32 reg;
	4199	+ u8 lanes = 1;
3275	4200	u8 speed;
	4201	+ struct dwc3_vendor *vdwc = container_of(dwc, struct dwc3_vendor, dwc);
	4202	+
	4203	+ if (!vdwc->softconnect)
	4204	+ return;
3276	4205
3277	4206	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
3278	4207	speed = reg & DWC3_DSTS_CONNECTSPD;
3279	4208	dwc->speed = speed;
	4209	+
	4210	+ if (DWC3_IP_IS(DWC32))
	4211	+ lanes = DWC3_DSTS_CONNLANES(reg) + 1;
	4212	+
	4213	+ dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
3280	4214
3281	4215	/*
3282	4216	* RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
..	..	@@ -3290,8 +4224,13 @@
3290	4224	switch (speed) {
3291	4225	case DWC3_DSTS_SUPERSPEED_PLUS:
3292	4226	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3293		- dwc->gadget.ep0->maxpacket = 512;
3294		- dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
	4227	+ dwc->gadget->ep0->maxpacket = 512;
	4228	+ dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
	4229	+
	4230	+ if (lanes > 1)
	4231	+ dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
	4232	+ else
	4233	+ dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
3295	4234	break;
3296	4235	case DWC3_DSTS_SUPERSPEED:
3297	4236	/*
..	..	@@ -3307,35 +4246,41 @@
3307	4246	* STAR#9000483510: RTL: SS : USB3 reset event may
3308	4247	* not be generated always when the link enters poll
3309	4248	*/
3310		- if (dwc->revision < DWC3_REVISION_190A)
	4249	+ if (DWC3_VER_IS_PRIOR(DWC3, 190A))
3311	4250	dwc3_gadget_reset_interrupt(dwc);
3312	4251
3313	4252	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3314		- dwc->gadget.ep0->maxpacket = 512;
3315		- dwc->gadget.speed = USB_SPEED_SUPER;
	4253	+ dwc->gadget->ep0->maxpacket = 512;
	4254	+ dwc->gadget->speed = USB_SPEED_SUPER;
	4255	+
	4256	+ if (lanes > 1) {
	4257	+ dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
	4258	+ dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
	4259	+ }
3316	4260	break;
3317	4261	case DWC3_DSTS_HIGHSPEED:
3318	4262	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3319		- dwc->gadget.ep0->maxpacket = 64;
3320		- dwc->gadget.speed = USB_SPEED_HIGH;
	4263	+ dwc->gadget->ep0->maxpacket = 64;
	4264	+ dwc->gadget->speed = USB_SPEED_HIGH;
3321	4265	break;
3322	4266	case DWC3_DSTS_FULLSPEED:
3323	4267	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3324		- dwc->gadget.ep0->maxpacket = 64;
3325		- dwc->gadget.speed = USB_SPEED_FULL;
	4268	+ dwc->gadget->ep0->maxpacket = 64;
	4269	+ dwc->gadget->speed = USB_SPEED_FULL;
3326	4270	break;
3327	4271	case DWC3_DSTS_LOWSPEED:
3328	4272	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
3329		- dwc->gadget.ep0->maxpacket = 8;
3330		- dwc->gadget.speed = USB_SPEED_LOW;
	4273	+ dwc->gadget->ep0->maxpacket = 8;
	4274	+ dwc->gadget->speed = USB_SPEED_LOW;
3331	4275	break;
3332	4276	}
3333	4277
3334		- dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
	4278	+ dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
3335	4279
3336	4280	/* Enable USB2 LPM Capability */
3337	4281
3338		- if ((dwc->revision > DWC3_REVISION_194A) &&
	4282	+ if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
	4283	+ !dwc->usb2_gadget_lpm_disable &&
3339	4284	(speed != DWC3_DSTS_SUPERSPEED) &&
3340	4285	(speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
3341	4286	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
..	..	@@ -3345,7 +4290,8 @@
3345	4290	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3346	4291	reg &= ~(DWC3_DCTL_HIRD_THRES_MASK \| DWC3_DCTL_L1_HIBER_EN);
3347	4292
3348		- reg \|= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
	4293	+ reg \|= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold \|
	4294	+ (dwc->is_utmi_l1_suspend << 4));
3349	4295
3350	4296	/*
3351	4297	* When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
..	..	@@ -3353,23 +4299,24 @@
3353	4299	* BESL value in the LPM token is less than or equal to LPM
3354	4300	* NYET threshold.
3355	4301	*/
3356		- WARN_ONCE(dwc->revision < DWC3_REVISION_240A
3357		- && dwc->has_lpm_erratum,
	4302	+ WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
3358	4303	"LPM Erratum not available on dwc3 revisions < 2.40a\n");
3359	4304
3360		- if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
	4305	+ if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
3361	4306	reg \|= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
3362	4307
3363		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	4308	+ dwc3_gadget_dctl_write_safe(dwc, reg);
3364	4309	} else {
	4310	+ if (dwc->usb2_gadget_lpm_disable) {
	4311	+ reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	4312	+ reg &= ~DWC3_DCFG_LPM_CAP;
	4313	+ dwc3_writel(dwc->regs, DWC3_DCFG, reg);
	4314	+ }
	4315	+
3365	4316	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3366	4317	reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
3367		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	4318	+ dwc3_gadget_dctl_write_safe(dwc, reg);
3368	4319	}
3369		-
3370		-#ifdef CONFIG_ARCH_ROCKCHIP
3371		- dwc3_gadget_resize_tx_fifos(dwc);
3372		-#endif
3373	4320
3374	4321	dep = dwc->eps[0];
3375	4322	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
..	..	@@ -3394,21 +4341,18 @@
3394	4341	*/
3395	4342	}
3396	4343
3397		-static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
	4344	+static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
3398	4345	{
3399		- enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3400	4346	/*
3401	4347	* TODO take core out of low power mode when that's
3402	4348	* implemented.
3403	4349	*/
3404	4350
3405		- if (dwc->gadget_driver && dwc->gadget_driver->resume && dwc->uwk_en) {
	4351	+ if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3406	4352	spin_unlock(&dwc->lock);
3407		- dwc->gadget_driver->resume(&dwc->gadget);
	4353	+ dwc->gadget_driver->resume(dwc->gadget);
3408	4354	spin_lock(&dwc->lock);
3409	4355	}
3410		-
3411		- dwc->link_state = next;
3412	4356	}
3413	4357
3414	4358	static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
..	..	@@ -3435,7 +4379,7 @@
3435	4379	* operational mode
3436	4380	*/
3437	4381	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
3438		- if ((dwc->revision < DWC3_REVISION_250A) &&
	4382	+ if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
3439	4383	(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
3440	4384	if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
3441	4385	(next == DWC3_LINK_STATE_RESUME)) {
..	..	@@ -3461,7 +4405,7 @@
3461	4405	* STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
3462	4406	* core send LGO_Ux entering U0
3463	4407	*/
3464		- if (dwc->revision < DWC3_REVISION_183A) {
	4408	+ if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3465	4409	if (next == DWC3_LINK_STATE_U0) {
3466	4410	u32 u1u2;
3467	4411	u32 reg;
..	..	@@ -3480,7 +4424,7 @@
3480	4424
3481	4425	reg &= ~u1u2;
3482	4426
3483		- dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	4427	+ dwc3_gadget_dctl_write_safe(dwc, reg);
3484	4428	break;
3485	4429	default:
3486	4430	/* do nothing */
..	..	@@ -3514,8 +4458,7 @@
3514	4458	{
3515	4459	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3516	4460
3517		- if (dwc->link_state != next && next == DWC3_LINK_STATE_U3 &&
3518		- dwc->uwk_en)
	4461	+ if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
3519	4462	dwc3_suspend_gadget(dwc);
3520	4463
3521	4464	dwc->link_state = next;
..	..	@@ -3561,8 +4504,7 @@
3561	4504	dwc3_gadget_conndone_interrupt(dwc);
3562	4505	break;
3563	4506	case DWC3_DEVICE_EVENT_WAKEUP:
3564		- dev_dbg(dwc->dev, "device wakeup\n");
3565		- dwc3_gadget_wakeup_interrupt(dwc, event->event_info);
	4507	+ dwc3_gadget_wakeup_interrupt(dwc);
3566	4508	break;
3567	4509	case DWC3_DEVICE_EVENT_HIBER_REQ:
3568	4510	if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
..	..	@@ -3574,18 +4516,10 @@
3574	4516	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3575	4517	dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3576	4518	break;
3577		- case DWC3_DEVICE_EVENT_EOPF:
	4519	+ case DWC3_DEVICE_EVENT_SUSPEND:
3578	4520	/* It changed to be suspend event for version 2.30a and above */
3579		- if (dwc->revision >= DWC3_REVISION_230A) {
3580		- /*
3581		- * Ignore suspend event until the gadget enters into
3582		- * USB_STATE_CONFIGURED state.
3583		- */
3584		- dev_dbg(dwc->dev, "device suspend\n");
3585		- if (dwc->gadget.state >= USB_STATE_CONFIGURED)
3586		- dwc3_gadget_suspend_interrupt(dwc,
3587		- event->event_info);
3588		- }
	4521	+ if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
	4522	+ dwc3_gadget_suspend_interrupt(dwc, event->event_info);
3589	4523	break;
3590	4524	case DWC3_DEVICE_EVENT_SOF:
3591	4525	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
..	..	@@ -3615,7 +4549,6 @@
3615	4549	struct dwc3 *dwc = evt->dwc;
3616	4550	irqreturn_t ret = IRQ_NONE;
3617	4551	int left;
3618		- u32 reg;
3619	4552
3620	4553	left = evt->count;
3621	4554
..	..	@@ -3643,18 +4576,19 @@
3643	4576	}
3644	4577
3645	4578	evt->count = 0;
3646		- evt->flags &= ~DWC3_EVENT_PENDING;
3647	4579	ret = IRQ_HANDLED;
3648	4580
3649	4581	/* Unmask interrupt */
3650		- reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3651		- reg &= ~DWC3_GEVNTSIZ_INTMASK;
3652		- dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
	4582	+ dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
	4583	+ DWC3_GEVNTSIZ_SIZE(evt->length));
3653	4584
3654	4585	if (dwc->imod_interval) {
3655	4586	dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3656	4587	dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3657	4588	}
	4589	+
	4590	+ /* Keep the clearing of DWC3_EVENT_PENDING at the end */
	4591	+ evt->flags &= ~DWC3_EVENT_PENDING;
3658	4592
3659	4593	return ret;
3660	4594	}
..	..	@@ -3680,12 +4614,16 @@
3680	4614	struct dwc3 *dwc = evt->dwc;
3681	4615	u32 amount;
3682	4616	u32 count;
3683		- u32 reg;
3684	4617
3685	4618	if (pm_runtime_suspended(dwc->dev)) {
	4619	+ dwc->pending_events = true;
	4620	+ /*
	4621	+ * Trigger runtime resume. The get() function will be balanced
	4622	+ * after processing the pending events in dwc3_process_pending
	4623	+ * events().
	4624	+ */
3686	4625	pm_runtime_get(dwc->dev);
3687	4626	disable_irq_nosync(dwc->irq_gadget);
3688		- dwc->pending_events = true;
3689	4627	return IRQ_HANDLED;
3690	4628	}
3691	4629
..	..	@@ -3707,9 +4645,8 @@
3707	4645	evt->flags \|= DWC3_EVENT_PENDING;
3708	4646
3709	4647	/* Mask interrupt */
3710		- reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3711		- reg \|= DWC3_GEVNTSIZ_INTMASK;
3712		- dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
	4648	+ dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
	4649	+ DWC3_GEVNTSIZ_INTMASK \| DWC3_GEVNTSIZ_SIZE(evt->length));
3713	4650
3714	4651	amount = min(count, evt->length - evt->lpos);
3715	4652	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
..	..	@@ -3734,14 +4671,14 @@
3734	4671	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3735	4672	int irq;
3736	4673
3737		- irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
	4674	+ irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
3738	4675	if (irq > 0)
3739	4676	goto out;
3740	4677
3741	4678	if (irq == -EPROBE_DEFER)
3742	4679	goto out;
3743	4680
3744		- irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
	4681	+ irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
3745	4682	if (irq > 0)
3746	4683	goto out;
3747	4684
..	..	@@ -3752,14 +4689,18 @@
3752	4689	if (irq > 0)
3753	4690	goto out;
3754	4691
3755		- if (irq != -EPROBE_DEFER)
3756		- dev_err(dwc->dev, "missing peripheral IRQ\n");
3757		-
3758	4692	if (!irq)
3759	4693	irq = -EINVAL;
3760	4694
3761	4695	out:
3762	4696	return irq;
	4697	+}
	4698	+
	4699	+static void dwc_gadget_release(struct device *dev)
	4700	+{
	4701	+ struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
	4702	+
	4703	+ kfree(gadget);
3763	4704	}
3764	4705
3765	4706	/**
..	..	@@ -3772,6 +4713,7 @@
3772	4713	{
3773	4714	int ret;
3774	4715	int irq;
	4716	+ struct device *dev;
3775	4717
3776	4718	irq = dwc3_gadget_get_irq(dwc);
3777	4719	if (irq < 0) {
..	..	@@ -3804,17 +4746,22 @@
3804	4746	}
3805	4747
3806	4748	init_completion(&dwc->ep0_in_setup);
3807		- init_completion(&dwc->discon_done);
	4749	+ dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
	4750	+ if (!dwc->gadget) {
	4751	+ ret = -ENOMEM;
	4752	+ goto err3;
	4753	+ }
3808	4754
3809		- dwc->gadget.ops = &dwc3_gadget_ops;
3810		- dwc->gadget.speed = USB_SPEED_UNKNOWN;
3811		- dwc->gadget.sg_supported = true;
3812		- dwc->gadget.name = "dwc3-gadget";
3813		-#ifdef CONFIG_ARCH_ROCKCHIP
3814		- dwc->gadget.lpm_capable = false;
3815		-#else
3816		- dwc->gadget.lpm_capable = true;
3817		-#endif
	4755	+
	4756	+ usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
	4757	+ dev = &dwc->gadget->dev;
	4758	+ dev->platform_data = dwc;
	4759	+ dwc->gadget->ops = &dwc3_gadget_ops;
	4760	+ dwc->gadget->speed = USB_SPEED_UNKNOWN;
	4761	+ dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
	4762	+ dwc->gadget->sg_supported = true;
	4763	+ dwc->gadget->name = "dwc3-gadget";
	4764	+ dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable;
3818	4765
3819	4766	/*
3820	4767	* FIXME We might be setting max_speed to <SUPER, however versions
..	..	@@ -3832,12 +4779,13 @@
3832	4779	* is less than super speed because we don't have means, yet, to tell
3833	4780	* composite.c that we are USB 2.0 + LPM ECN.
3834	4781	*/
3835		- if (dwc->revision < DWC3_REVISION_220A &&
	4782	+ if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
3836	4783	!dwc->dis_metastability_quirk)
3837	4784	dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3838	4785	dwc->revision);
3839	4786
3840		- dwc->gadget.max_speed = dwc->maximum_speed;
	4787	+ dwc->gadget->max_speed = dwc->maximum_speed;
	4788	+ dwc->gadget->max_ssp_rate = dwc->max_ssp_rate;
3841	4789
3842	4790	/*
3843	4791	* REVISIT: Here we should clear all pending IRQs to be
..	..	@@ -3846,21 +4794,26 @@
3846	4794
3847	4795	ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3848	4796	if (ret)
3849		- goto err3;
3850		-
3851		- ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3852		- if (ret) {
3853		- dev_err(dwc->dev, "failed to register udc\n");
3854	4797	goto err4;
	4798	+
	4799	+ ret = usb_add_gadget(dwc->gadget);
	4800	+ if (ret) {
	4801	+ dev_err(dwc->dev, "failed to add gadget\n");
	4802	+ goto err5;
3855	4803	}
3856	4804
3857		- dwc3_gadget_set_speed(&dwc->gadget, dwc->maximum_speed);
	4805	+ if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
	4806	+ dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);
	4807	+ else
	4808	+ dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
3858	4809
3859	4810	return 0;
3860	4811
3861		-err4:
	4812	+err5:
3862	4813	dwc3_gadget_free_endpoints(dwc);
3863		-
	4814	+err4:
	4815	+ usb_put_gadget(dwc->gadget);
	4816	+ dwc->gadget = NULL;
3864	4817	err3:
3865	4818	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3866	4819	dwc->bounce_addr);
..	..	@@ -3880,8 +4833,12 @@
3880	4833
3881	4834	void dwc3_gadget_exit(struct dwc3 *dwc)
3882	4835	{
3883		- usb_del_gadget_udc(&dwc->gadget);
	4836	+ if (!dwc->gadget)
	4837	+ return;
	4838	+
	4839	+ usb_del_gadget(dwc->gadget);
3884	4840	dwc3_gadget_free_endpoints(dwc);
	4841	+ usb_put_gadget(dwc->gadget);
3885	4842	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3886	4843	dwc->bounce_addr);
3887	4844	kfree(dwc->setup_buf);
..	..	@@ -3891,21 +4848,27 @@
3891	4848
3892	4849	int dwc3_gadget_suspend(struct dwc3 *dwc)
3893	4850	{
	4851	+ unsigned long flags;
	4852	+
3894	4853	if (!dwc->gadget_driver)
3895	4854	return 0;
3896	4855
3897	4856	dwc3_gadget_run_stop(dwc, false, false);
	4857	+
	4858	+ spin_lock_irqsave(&dwc->lock, flags);
3898	4859	dwc3_disconnect_gadget(dwc);
3899	4860	__dwc3_gadget_stop(dwc);
	4861	+ spin_unlock_irqrestore(&dwc->lock, flags);
3900	4862
3901	4863	return 0;
3902	4864	}
3903	4865
3904	4866	int dwc3_gadget_resume(struct dwc3 *dwc)
3905	4867	{
	4868	+ struct dwc3_vendor *vdwc = container_of(dwc, struct dwc3_vendor, dwc);
3906	4869	int ret;
3907	4870
3908		- if (!dwc->gadget_driver)
	4871	+ if (!dwc->gadget_driver \|\| !vdwc->softconnect)
3909	4872	return 0;
3910	4873
3911	4874	ret = __dwc3_gadget_start(dwc);
..	..	@@ -3929,6 +4892,8 @@
3929	4892	{
3930	4893	if (dwc->pending_events) {
3931	4894	dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
	4895	+ dwc3_thread_interrupt(dwc->irq_gadget, dwc->ev_buf);
	4896	+ pm_runtime_put(dwc->dev);
3932	4897	dwc->pending_events = false;
3933	4898	enable_irq(dwc->irq_gadget);
3934	4899	}