~hc/RK356X_SDK_RELEASE.git

..	..	@@ -57,7 +57,10 @@
57	57	#include <linux/dma-mapping.h>
58	58	#include "xhci.h"
59	59	#include "xhci-trace.h"
60		-#include "xhci-mtk.h"
	60	+
	61	+static int queue_command(struct xhci_hcd xhci, struct xhci_command cmd,
	62	+ u32 field1, u32 field2,
	63	+ u32 field3, u32 field4, bool command_must_succeed);
61	64
62	65	/*
63	66	* Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
..	..	@@ -76,6 +79,7 @@
76	79	return 0;
77	80	return seg->dma + (segment_offset * sizeof(*trb));
78	81	}
	82	+EXPORT_SYMBOL_GPL(xhci_trb_virt_to_dma);
79	83
80	84	static bool trb_is_noop(union xhci_trb *trb)
81	85	{
..	..	@@ -151,10 +155,11 @@
151	155
152	156	/*
153	157	* See Cycle bit rules. SW is the consumer for the event ring only.
154		- * Don't make a ring full of link TRBs. That would be dumb and this would loop.
155	158	*/
156	159	void inc_deq(struct xhci_hcd xhci, struct xhci_ring ring)
157	160	{
	161	+ unsigned int link_trb_count = 0;
	162	+
158	163	/* event ring doesn't have link trbs, check for last trb */
159	164	if (ring->type == TYPE_EVENT) {
160	165	if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
..	..	@@ -170,14 +175,23 @@
170	175
171	176	/* All other rings have link trbs */
172	177	if (!trb_is_link(ring->dequeue)) {
173		- ring->dequeue++;
174		- ring->num_trbs_free++;
	178	+ if (last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
	179	+ xhci_warn(xhci, "Missing link TRB at end of segment\n");
	180	+ } else {
	181	+ ring->dequeue++;
	182	+ ring->num_trbs_free++;
	183	+ }
175	184	}
	185	+
176	186	while (trb_is_link(ring->dequeue)) {
177	187	ring->deq_seg = ring->deq_seg->next;
178	188	ring->dequeue = ring->deq_seg->trbs;
179		- }
180	189
	190	+ if (link_trb_count++ > ring->num_segs) {
	191	+ xhci_warn(xhci, "Ring is an endless link TRB loop\n");
	192	+ break;
	193	+ }
	194	+ }
181	195	out:
182	196	trace_xhci_inc_deq(ring);
183	197
..	..	@@ -186,7 +200,6 @@
186	200
187	201	/*
188	202	* See Cycle bit rules. SW is the consumer for the event ring only.
189		- * Don't make a ring full of link TRBs. That would be dumb and this would loop.
190	203	*
191	204	* If we've just enqueued a TRB that is in the middle of a TD (meaning the
192	205	* chain bit is set), then set the chain bit in all the following link TRBs.
..	..	@@ -206,11 +219,18 @@
206	219	{
207	220	u32 chain;
208	221	union xhci_trb *next;
	222	+ unsigned int link_trb_count = 0;
209	223
210	224	chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
211	225	/* If this is not event ring, there is one less usable TRB */
212	226	if (!trb_is_link(ring->enqueue))
213	227	ring->num_trbs_free--;
	228	+
	229	+ if (last_trb_on_seg(ring->enq_seg, ring->enqueue)) {
	230	+ xhci_err(xhci, "Tried to move enqueue past ring segment\n");
	231	+ return;
	232	+ }
	233	+
214	234	next = ++(ring->enqueue);
215	235
216	236	/* Update the dequeue pointer further if that was a link TRB */
..	..	@@ -247,9 +267,34 @@
247	267	ring->enq_seg = ring->enq_seg->next;
248	268	ring->enqueue = ring->enq_seg->trbs;
249	269	next = ring->enqueue;
	270	+
	271	+ if (link_trb_count++ > ring->num_segs) {
	272	+ xhci_warn(xhci, "%s: Ring link TRB loop\n", __func__);
	273	+ break;
	274	+ }
250	275	}
251	276
252	277	trace_xhci_inc_enq(ring);
	278	+}
	279	+
	280	+static int xhci_num_trbs_to(struct xhci_segment start_seg, union xhci_trb start,
	281	+ struct xhci_segment end_seg, union xhci_trb end,
	282	+ unsigned int num_segs)
	283	+{
	284	+ union xhci_trb *last_on_seg;
	285	+ int num = 0;
	286	+ int i = 0;
	287	+
	288	+ do {
	289	+ if (start_seg == end_seg && end >= start)
	290	+ return num + (end - start);
	291	+ last_on_seg = &start_seg->trbs[TRBS_PER_SEGMENT - 1];
	292	+ num += last_on_seg - start;
	293	+ start_seg = start_seg->next;
	294	+ start = start_seg->trbs;
	295	+ } while (i++ <= num_segs);
	296	+
	297	+ return -EINVAL;
253	298	}
254	299
255	300	/*
..	..	@@ -280,10 +325,14 @@
280	325	return;
281	326
282	327	xhci_dbg(xhci, "// Ding dong!\n");
	328	+
	329	+ trace_xhci_ring_host_doorbell(0, DB_VALUE_HOST);
	330	+
283	331	writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]);
284	332	/* Flush PCI posted writes */
285	333	readl(&xhci->dba->doorbell[0]);
286	334	}
	335	+EXPORT_SYMBOL_GPL(xhci_ring_cmd_db);
287	336
288	337	static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay)
289	338	{
..	..	@@ -412,12 +461,14 @@
412	461	* stream once the endpoint is on the HW schedule.
413	462	*/
414	463	if ((ep_state & EP_STOP_CMD_PENDING) \|\| (ep_state & SET_DEQ_PENDING) \|\|
415		- (ep_state & EP_HALTED))
	464	+ (ep_state & EP_HALTED) \|\| (ep_state & EP_CLEARING_TT))
416	465	return;
	466	+
	467	+ trace_xhci_ring_ep_doorbell(slot_id, DB_VALUE(ep_index, stream_id));
	468	+
417	469	writel(DB_VALUE(ep_index, stream_id), db_addr);
418		- /* The CPU has better things to do at this point than wait for a
419		- * write-posting flush. It'll get there soon enough.
420		- */
	470	+ /* flush the write */
	471	+ readl(db_addr);
421	472	}
422	473
423	474	/* Ring the doorbell for any rings with pending URBs */
..	..	@@ -446,6 +497,13 @@
446	497	}
447	498	}
448	499
	500	+void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
	501	+ unsigned int slot_id,
	502	+ unsigned int ep_index)
	503	+{
	504	+ ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
	505	+}
	506	+
449	507	static struct xhci_virt_ep xhci_get_virt_ep(struct xhci_hcd xhci,
450	508	unsigned int slot_id,
451	509	unsigned int ep_index)
..	..	@@ -466,6 +524,26 @@
466	524	return &xhci->devs[slot_id]->eps[ep_index];
467	525	}
468	526
	527	+static struct xhci_ring xhci_virt_ep_to_ring(struct xhci_hcd xhci,
	528	+ struct xhci_virt_ep *ep,
	529	+ unsigned int stream_id)
	530	+{
	531	+ /* common case, no streams */
	532	+ if (!(ep->ep_state & EP_HAS_STREAMS))
	533	+ return ep->ring;
	534	+
	535	+ if (!ep->stream_info)
	536	+ return NULL;
	537	+
	538	+ if (stream_id == 0 \|\| stream_id >= ep->stream_info->num_streams) {
	539	+ xhci_warn(xhci, "Invalid stream_id %u request for slot_id %u ep_index %u\n",
	540	+ stream_id, ep->vdev->slot_id, ep->ep_index);
	541	+ return NULL;
	542	+ }
	543	+
	544	+ return ep->stream_info->stream_rings[stream_id];
	545	+}
	546	+
469	547	/* Get the right ring for the given slot_id, ep_index and stream_id.
470	548	* If the endpoint supports streams, boundary check the URB's stream ID.
471	549	* If the endpoint doesn't support streams, return the singular endpoint ring.
..	..	@@ -480,29 +558,7 @@
480	558	if (!ep)
481	559	return NULL;
482	560
483		- /* Common case: no streams */
484		- if (!(ep->ep_state & EP_HAS_STREAMS))
485		- return ep->ring;
486		-
487		- if (stream_id == 0) {
488		- xhci_warn(xhci,
489		- "WARN: Slot ID %u, ep index %u has streams, "
490		- "but URB has no stream ID.\n",
491		- slot_id, ep_index);
492		- return NULL;
493		- }
494		-
495		- if (stream_id < ep->stream_info->num_streams)
496		- return ep->stream_info->stream_rings[stream_id];
497		-
498		- xhci_warn(xhci,
499		- "WARN: Slot ID %u, ep index %u has "
500		- "stream IDs 1 to %u allocated, "
501		- "but stream ID %u is requested.\n",
502		- slot_id, ep_index,
503		- ep->stream_info->num_streams - 1,
504		- stream_id);
505		- return NULL;
	561	+ return xhci_virt_ep_to_ring(xhci, ep, stream_id);
506	562	}
507	563
508	564
..	..	@@ -529,56 +585,78 @@
529	585	return le64_to_cpu(ep_ctx->deq);
530	586	}
531	587
532		-/*
533		- * Move the xHC's endpoint ring dequeue pointer past cur_td.
534		- * Record the new state of the xHC's endpoint ring dequeue segment,
535		- * dequeue pointer, stream id, and new consumer cycle state in state.
536		- * Update our internal representation of the ring's dequeue pointer.
537		- *
538		- * We do this in three jumps:
539		- * - First we update our new ring state to be the same as when the xHC stopped.
540		- * - Then we traverse the ring to find the segment that contains
541		- * the last TRB in the TD. We toggle the xHC's new cycle state when we pass
542		- * any link TRBs with the toggle cycle bit set.
543		- * - Finally we move the dequeue state one TRB further, toggling the cycle bit
544		- * if we've moved it past a link TRB with the toggle cycle bit set.
545		- *
546		- * Some of the uses of xhci_generic_trb are grotty, but if they're done
547		- * with correct __le32 accesses they should work fine. Only users of this are
548		- * in here.
549		- */
550		-void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
551		- unsigned int slot_id, unsigned int ep_index,
552		- unsigned int stream_id, struct xhci_td *cur_td,
553		- struct xhci_dequeue_state *state)
	588	+static int xhci_move_dequeue_past_td(struct xhci_hcd *xhci,
	589	+ unsigned int slot_id, unsigned int ep_index,
	590	+ unsigned int stream_id, struct xhci_td *td)
554	591	{
555	592	struct xhci_virt_device *dev = xhci->devs[slot_id];
556	593	struct xhci_virt_ep *ep = &dev->eps[ep_index];
557	594	struct xhci_ring *ep_ring;
	595	+ struct xhci_command *cmd;
558	596	struct xhci_segment *new_seg;
	597	+ struct xhci_segment *halted_seg = NULL;
559	598	union xhci_trb *new_deq;
	599	+ int new_cycle;
	600	+ union xhci_trb *halted_trb;
	601	+ int index = 0;
560	602	dma_addr_t addr;
561	603	u64 hw_dequeue;
562	604	bool cycle_found = false;
563	605	bool td_last_trb_found = false;
	606	+ u32 trb_sct = 0;
	607	+ int ret;
564	608
565	609	ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,
566	610	ep_index, stream_id);
567	611	if (!ep_ring) {
568		- xhci_warn(xhci, "WARN can't find new dequeue state "
569		- "for invalid stream ID %u.\n",
570		- stream_id);
571		- return;
	612	+ xhci_warn(xhci, "WARN can't find new dequeue, invalid stream ID %u\n",
	613	+ stream_id);
	614	+ return -ENODEV;
572	615	}
573		- /* Dig out the cycle state saved by the xHC during the stop ep cmd */
574		- xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
575		- "Finding endpoint context");
	616	+ /*
	617	+ * A cancelled TD can complete with a stall if HW cached the trb.
	618	+ * In this case driver can't find td, but if the ring is empty we
	619	+ * can move the dequeue pointer to the current enqueue position.
	620	+ * We shouldn't hit this anymore as cached cancelled TRBs are given back
	621	+ * after clearing the cache, but be on the safe side and keep it anyway
	622	+ */
	623	+ if (!td) {
	624	+ if (list_empty(&ep_ring->td_list)) {
	625	+ new_seg = ep_ring->enq_seg;
	626	+ new_deq = ep_ring->enqueue;
	627	+ new_cycle = ep_ring->cycle_state;
	628	+ xhci_dbg(xhci, "ep ring empty, Set new dequeue = enqueue");
	629	+ goto deq_found;
	630	+ } else {
	631	+ xhci_warn(xhci, "Can't find new dequeue state, missing td\n");
	632	+ return -EINVAL;
	633	+ }
	634	+ }
576	635
577	636	hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id);
578	637	new_seg = ep_ring->deq_seg;
579	638	new_deq = ep_ring->dequeue;
580		- state->new_cycle_state = hw_dequeue & 0x1;
581		- state->stream_id = stream_id;
	639	+
	640	+ /*
	641	+ * Quirk: xHC write-back of the DCS field in the hardware dequeue
	642	+ * pointer is wrong - use the cycle state of the TRB pointed to by
	643	+ * the dequeue pointer.
	644	+ */
	645	+ if (xhci->quirks & XHCI_EP_CTX_BROKEN_DCS &&
	646	+ !(ep->ep_state & EP_HAS_STREAMS))
	647	+ halted_seg = trb_in_td(xhci, td->start_seg,
	648	+ td->first_trb, td->last_trb,
	649	+ hw_dequeue & ~0xf, false);
	650	+ if (halted_seg) {
	651	+ index = ((dma_addr_t)(hw_dequeue & ~0xf) - halted_seg->dma) /
	652	+ sizeof(*halted_trb);
	653	+ halted_trb = &halted_seg->trbs[index];
	654	+ new_cycle = halted_trb->generic.field[3] & 0x1;
	655	+ xhci_dbg(xhci, "Endpoint DCS = %d TRB index = %d cycle = %d\n",
	656	+ (u8)(hw_dequeue & 0x1), index, new_cycle);
	657	+ } else {
	658	+ new_cycle = hw_dequeue & 0x1;
	659	+ }
582	660
583	661	/*
584	662	* We want to find the pointer, segment and cycle state of the new trb
..	..	@@ -593,39 +671,71 @@
593	671	if (td_last_trb_found)
594	672	break;
595	673	}
596		- if (new_deq == cur_td->last_trb)
	674	+ if (new_deq == td->last_trb)
597	675	td_last_trb_found = true;
598	676
599	677	if (cycle_found && trb_is_link(new_deq) &&
600	678	link_trb_toggles_cycle(new_deq))
601		- state->new_cycle_state ^= 0x1;
	679	+ new_cycle ^= 0x1;
602	680
603	681	next_trb(xhci, ep_ring, &new_seg, &new_deq);
604	682
605	683	/* Search wrapped around, bail out */
606	684	if (new_deq == ep->ring->dequeue) {
607	685	xhci_err(xhci, "Error: Failed finding new dequeue state\n");
608		- state->new_deq_seg = NULL;
609		- state->new_deq_ptr = NULL;
610		- return;
	686	+ return -EINVAL;
611	687	}
612	688
613	689	} while (!cycle_found \|\| !td_last_trb_found);
614	690
615		- state->new_deq_seg = new_seg;
616		- state->new_deq_ptr = new_deq;
	691	+deq_found:
617	692
618	693	/* Don't update the ring cycle state for the producer (us). */
619		- xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
620		- "Cycle state = 0x%x", state->new_cycle_state);
	694	+ addr = xhci_trb_virt_to_dma(new_seg, new_deq);
	695	+ if (addr == 0) {
	696	+ xhci_warn(xhci, "Can't find dma of new dequeue ptr\n");
	697	+ xhci_warn(xhci, "deq seg = %p, deq ptr = %p\n", new_seg, new_deq);
	698	+ return -EINVAL;
	699	+ }
	700	+
	701	+ if ((ep->ep_state & SET_DEQ_PENDING)) {
	702	+ xhci_warn(xhci, "Set TR Deq already pending, don't submit for 0x%pad\n",
	703	+ &addr);
	704	+ return -EBUSY;
	705	+ }
	706	+
	707	+ /* This function gets called from contexts where it cannot sleep */
	708	+ cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC);
	709	+ if (!cmd) {
	710	+ xhci_warn(xhci, "Can't alloc Set TR Deq cmd 0x%pad\n", &addr);
	711	+ return -ENOMEM;
	712	+ }
	713	+
	714	+ if (stream_id)
	715	+ trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
	716	+ ret = queue_command(xhci, cmd,
	717	+ lower_32_bits(addr) \| trb_sct \| new_cycle,
	718	+ upper_32_bits(addr),
	719	+ STREAM_ID_FOR_TRB(stream_id), SLOT_ID_FOR_TRB(slot_id) \|
	720	+ EP_ID_FOR_TRB(ep_index) \| TRB_TYPE(TRB_SET_DEQ), false);
	721	+ if (ret < 0) {
	722	+ xhci_free_command(xhci, cmd);
	723	+ return ret;
	724	+ }
	725	+ ep->queued_deq_seg = new_seg;
	726	+ ep->queued_deq_ptr = new_deq;
621	727
622	728	xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
623		- "New dequeue segment = %p (virtual)",
624		- state->new_deq_seg);
625		- addr = xhci_trb_virt_to_dma(state->new_deq_seg, state->new_deq_ptr);
626		- xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
627		- "New dequeue pointer = 0x%llx (DMA)",
628		- (unsigned long long) addr);
	729	+ "Set TR Deq ptr 0x%llx, cycle %u\n", addr, new_cycle);
	730	+
	731	+ /* Stop the TD queueing code from ringing the doorbell until
	732	+ * this command completes. The HC won't set the dequeue pointer
	733	+ * if the ring is running, and ringing the doorbell starts the
	734	+ * ring running.
	735	+ */
	736	+ ep->ep_state \|= SET_DEQ_PENDING;
	737	+ xhci_ring_cmd_db(xhci);
	738	+ return 0;
629	739	}
630	740
631	741	/* flip_cycle means flip the cycle bit of all but the first and last TRB.
..	..	@@ -680,10 +790,8 @@
680	790	}
681	791	xhci_urb_free_priv(urb_priv);
682	792	usb_hcd_unlink_urb_from_ep(hcd, urb);
683		- spin_unlock(&xhci->lock);
684	793	trace_xhci_urb_giveback(urb);
685	794	usb_hcd_giveback_urb(hcd, urb, status);
686		- spin_lock(&xhci->lock);
687	795	}
688	796
689	797	static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
..	..	@@ -720,6 +828,256 @@
720	828	seg->bounce_offs = 0;
721	829	}
722	830
	831	+static int xhci_td_cleanup(struct xhci_hcd xhci, struct xhci_td td,
	832	+ struct xhci_ring *ep_ring, int status)
	833	+{
	834	+ struct urb *urb = NULL;
	835	+
	836	+ /* Clean up the endpoint's TD list */
	837	+ urb = td->urb;
	838	+
	839	+ /* if a bounce buffer was used to align this td then unmap it */
	840	+ xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);
	841	+
	842	+ /* Do one last check of the actual transfer length.
	843	+ * If the host controller said we transferred more data than the buffer
	844	+ * length, urb->actual_length will be a very big number (since it's
	845	+ * unsigned). Play it safe and say we didn't transfer anything.
	846	+ */
	847	+ if (urb->actual_length > urb->transfer_buffer_length) {
	848	+ xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
	849	+ urb->transfer_buffer_length, urb->actual_length);
	850	+ urb->actual_length = 0;
	851	+ status = 0;
	852	+ }
	853	+ /* TD might be removed from td_list if we are giving back a cancelled URB */
	854	+ if (!list_empty(&td->td_list))
	855	+ list_del_init(&td->td_list);
	856	+ /* Giving back a cancelled URB, or if a slated TD completed anyway */
	857	+ if (!list_empty(&td->cancelled_td_list))
	858	+ list_del_init(&td->cancelled_td_list);
	859	+
	860	+ inc_td_cnt(urb);
	861	+ /* Giveback the urb when all the tds are completed */
	862	+ if (last_td_in_urb(td)) {
	863	+ if ((urb->actual_length != urb->transfer_buffer_length &&
	864	+ (urb->transfer_flags & URB_SHORT_NOT_OK)) \|\|
	865	+ (status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc)))
	866	+ xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
	867	+ urb, urb->actual_length,
	868	+ urb->transfer_buffer_length, status);
	869	+
	870	+ /* set isoc urb status to 0 just as EHCI, UHCI, and OHCI */
	871	+ if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
	872	+ status = 0;
	873	+ xhci_giveback_urb_in_irq(xhci, td, status);
	874	+ }
	875	+
	876	+ return 0;
	877	+}
	878	+
	879	+
	880	+/* Complete the cancelled URBs we unlinked from td_list. */
	881	+static void xhci_giveback_invalidated_tds(struct xhci_virt_ep *ep)
	882	+{
	883	+ struct xhci_ring *ring;
	884	+ struct xhci_td td, tmp_td;
	885	+
	886	+ list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list,
	887	+ cancelled_td_list) {
	888	+
	889	+ ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb);
	890	+
	891	+ if (td->cancel_status == TD_CLEARED) {
	892	+ xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n",
	893	+ __func__, td->urb);
	894	+ xhci_td_cleanup(ep->xhci, td, ring, td->status);
	895	+ } else {
	896	+ xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n",
	897	+ __func__, td->urb, td->cancel_status);
	898	+ }
	899	+ if (ep->xhci->xhc_state & XHCI_STATE_DYING)
	900	+ return;
	901	+ }
	902	+}
	903	+
	904	+static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id,
	905	+ unsigned int ep_index, enum xhci_ep_reset_type reset_type)
	906	+{
	907	+ struct xhci_command *command;
	908	+ int ret = 0;
	909	+
	910	+ command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
	911	+ if (!command) {
	912	+ ret = -ENOMEM;
	913	+ goto done;
	914	+ }
	915	+
	916	+ xhci_dbg(xhci, "%s-reset ep %u, slot %u\n",
	917	+ (reset_type == EP_HARD_RESET) ? "Hard" : "Soft",
	918	+ ep_index, slot_id);
	919	+
	920	+ ret = xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type);
	921	+done:
	922	+ if (ret)
	923	+ xhci_err(xhci, "ERROR queuing reset endpoint for slot %d ep_index %d, %d\n",
	924	+ slot_id, ep_index, ret);
	925	+ return ret;
	926	+}
	927	+
	928	+static int xhci_handle_halted_endpoint(struct xhci_hcd *xhci,
	929	+ struct xhci_virt_ep *ep, unsigned int stream_id,
	930	+ struct xhci_td *td,
	931	+ enum xhci_ep_reset_type reset_type)
	932	+{
	933	+ unsigned int slot_id = ep->vdev->slot_id;
	934	+ int err;
	935	+
	936	+ /*
	937	+ * Avoid resetting endpoint if link is inactive. Can cause host hang.
	938	+ * Device will be reset soon to recover the link so don't do anything
	939	+ */
	940	+ if (ep->vdev->flags & VDEV_PORT_ERROR)
	941	+ return -ENODEV;
	942	+
	943	+ /* add td to cancelled list and let reset ep handler take care of it */
	944	+ if (reset_type == EP_HARD_RESET) {
	945	+ ep->ep_state \|= EP_HARD_CLEAR_TOGGLE;
	946	+ if (td && list_empty(&td->cancelled_td_list)) {
	947	+ list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list);
	948	+ td->cancel_status = TD_HALTED;
	949	+ }
	950	+ }
	951	+
	952	+ if (ep->ep_state & EP_HALTED) {
	953	+ xhci_dbg(xhci, "Reset ep command for ep_index %d already pending\n",
	954	+ ep->ep_index);
	955	+ return 0;
	956	+ }
	957	+
	958	+ err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type);
	959	+ if (err)
	960	+ return err;
	961	+
	962	+ ep->ep_state \|= EP_HALTED;
	963	+
	964	+ xhci_ring_cmd_db(xhci);
	965	+
	966	+ return 0;
	967	+}
	968	+
	969	+/*
	970	+ * Fix up the ep ring first, so HW stops executing cancelled TDs.
	971	+ * We have the xHCI lock, so nothing can modify this list until we drop it.
	972	+ * We're also in the event handler, so we can't get re-interrupted if another
	973	+ * Stop Endpoint command completes.
	974	+ *
	975	+ * only call this when ring is not in a running state
	976	+ */
	977	+
	978	+static int xhci_invalidate_cancelled_tds(struct xhci_virt_ep *ep)
	979	+{
	980	+ struct xhci_hcd *xhci;
	981	+ struct xhci_td *td = NULL;
	982	+ struct xhci_td *tmp_td = NULL;
	983	+ struct xhci_td *cached_td = NULL;
	984	+ struct xhci_ring *ring;
	985	+ u64 hw_deq;
	986	+ unsigned int slot_id = ep->vdev->slot_id;
	987	+ int err;
	988	+
	989	+ xhci = ep->xhci;
	990	+
	991	+ list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) {
	992	+ xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
	993	+ "Removing canceled TD starting at 0x%llx (dma) in stream %u URB %p",
	994	+ (unsigned long long)xhci_trb_virt_to_dma(
	995	+ td->start_seg, td->first_trb),
	996	+ td->urb->stream_id, td->urb);
	997	+ list_del_init(&td->td_list);
	998	+ ring = xhci_urb_to_transfer_ring(xhci, td->urb);
	999	+ if (!ring) {
	1000	+ xhci_warn(xhci, "WARN Cancelled URB %p has invalid stream ID %u.\n",
	1001	+ td->urb, td->urb->stream_id);
	1002	+ continue;
	1003	+ }
	1004	+ /*
	1005	+ * If a ring stopped on the TD we need to cancel then we have to
	1006	+ * move the xHC endpoint ring dequeue pointer past this TD.
	1007	+ * Rings halted due to STALL may show hw_deq is past the stalled
	1008	+ * TD, but still require a set TR Deq command to flush xHC cache.
	1009	+ */
	1010	+ hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index,
	1011	+ td->urb->stream_id);
	1012	+ hw_deq &= ~0xf;
	1013	+
	1014	+ if (td->cancel_status == TD_HALTED \|\|
	1015	+ trb_in_td(xhci, td->start_seg, td->first_trb, td->last_trb, hw_deq, false)) {
	1016	+ switch (td->cancel_status) {
	1017	+ case TD_CLEARED: /* TD is already no-op */
	1018	+ case TD_CLEARING_CACHE: /* set TR deq command already queued */
	1019	+ break;
	1020	+ case TD_DIRTY: /* TD is cached, clear it */
	1021	+ case TD_HALTED:
	1022	+ td->cancel_status = TD_CLEARING_CACHE;
	1023	+ if (cached_td)
	1024	+ /* FIXME stream case, several stopped rings */
	1025	+ xhci_dbg(xhci,
	1026	+ "Move dq past stream %u URB %p instead of stream %u URB %p\n",
	1027	+ td->urb->stream_id, td->urb,
	1028	+ cached_td->urb->stream_id, cached_td->urb);
	1029	+ cached_td = td;
	1030	+ ring->num_trbs_free += td->num_trbs;
	1031	+ break;
	1032	+ }
	1033	+ } else {
	1034	+ td_to_noop(xhci, ring, td, false);
	1035	+ td->cancel_status = TD_CLEARED;
	1036	+ ring->num_trbs_free += td->num_trbs;
	1037	+ }
	1038	+ }
	1039	+
	1040	+ /* If there's no need to move the dequeue pointer then we're done */
	1041	+ if (!cached_td)
	1042	+ return 0;
	1043	+
	1044	+ err = xhci_move_dequeue_past_td(xhci, slot_id, ep->ep_index,
	1045	+ cached_td->urb->stream_id,
	1046	+ cached_td);
	1047	+ if (err) {
	1048	+ /* Failed to move past cached td, just set cached TDs to no-op */
	1049	+ list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) {
	1050	+ if (td->cancel_status != TD_CLEARING_CACHE)
	1051	+ continue;
	1052	+ xhci_dbg(xhci, "Failed to clear cancelled cached URB %p, mark clear anyway\n",
	1053	+ td->urb);
	1054	+ td_to_noop(xhci, ring, td, false);
	1055	+ td->cancel_status = TD_CLEARED;
	1056	+ }
	1057	+ }
	1058	+ return 0;
	1059	+}
	1060	+
	1061	+/*
	1062	+ * Returns the TD the endpoint ring halted on.
	1063	+ * Only call for non-running rings without streams.
	1064	+ */
	1065	+static struct xhci_td find_halted_td(struct xhci_virt_ep ep)
	1066	+{
	1067	+ struct xhci_td *td;
	1068	+ u64 hw_deq;
	1069	+
	1070	+ if (!list_empty(&ep->ring->td_list)) { /* Not streams compatible */
	1071	+ hw_deq = xhci_get_hw_deq(ep->xhci, ep->vdev, ep->ep_index, 0);
	1072	+ hw_deq &= ~0xf;
	1073	+ td = list_first_entry(&ep->ring->td_list, struct xhci_td, td_list);
	1074	+ if (trb_in_td(ep->xhci, td->start_seg, td->first_trb,
	1075	+ td->last_trb, hw_deq, false))
	1076	+ return td;
	1077	+ }
	1078	+ return NULL;
	1079	+}
	1080	+
723	1081	/*
724	1082	* When we get a command completion for a Stop Endpoint Command, we need to
725	1083	* unlink any cancelled TDs from the ring. There are two ways to do that:
..	..	@@ -731,142 +1089,90 @@
731	1089	* bit cleared) so that the HW will skip over them.
732	1090	*/
733	1091	static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
734		- union xhci_trb trb, struct xhci_event_cmd event)
	1092	+ union xhci_trb *trb, u32 comp_code)
735	1093	{
736	1094	unsigned int ep_index;
737		- struct xhci_ring *ep_ring;
738	1095	struct xhci_virt_ep *ep;
739		- struct xhci_td *cur_td = NULL;
740		- struct xhci_td *last_unlinked_td;
741	1096	struct xhci_ep_ctx *ep_ctx;
742		- struct xhci_virt_device *vdev;
743		- u64 hw_deq;
744		- struct xhci_dequeue_state deq_state;
	1097	+ struct xhci_td *td = NULL;
	1098	+ enum xhci_ep_reset_type reset_type;
	1099	+ struct xhci_command *command;
	1100	+ int err;
745	1101
746	1102	if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) {
747	1103	if (!xhci->devs[slot_id])
748		- xhci_warn(xhci, "Stop endpoint command "
749		- "completion for disabled slot %u\n",
750		- slot_id);
	1104	+ xhci_warn(xhci, "Stop endpoint command completion for disabled slot %u\n",
	1105	+ slot_id);
751	1106	return;
752	1107	}
753	1108
754		- memset(&deq_state, 0, sizeof(deq_state));
755	1109	ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
756		-
757	1110	ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
758	1111	if (!ep)
759	1112	return;
760	1113
761		- vdev = xhci->devs[slot_id];
762		- ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
	1114	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
	1115	+
763	1116	trace_xhci_handle_cmd_stop_ep(ep_ctx);
764	1117
765		- last_unlinked_td = list_last_entry(&ep->cancelled_td_list,
766		- struct xhci_td, cancelled_td_list);
767		-
768		- if (list_empty(&ep->cancelled_td_list)) {
769		- xhci_stop_watchdog_timer_in_irq(xhci, ep);
770		- ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
771		- return;
772		- }
773		-
774		- /* Fix up the ep ring first, so HW stops executing cancelled TDs.
775		- * We have the xHCI lock, so nothing can modify this list until we drop
776		- * it. We're also in the event handler, so we can't get re-interrupted
777		- * if another Stop Endpoint command completes
	1118	+ if (comp_code == COMP_CONTEXT_STATE_ERROR) {
	1119	+ /*
	1120	+ * If stop endpoint command raced with a halting endpoint we need to
	1121	+ * reset the host side endpoint first.
	1122	+ * If the TD we halted on isn't cancelled the TD should be given back
	1123	+ * with a proper error code, and the ring dequeue moved past the TD.
	1124	+ * If streams case we can't find hw_deq, or the TD we halted on so do a
	1125	+ * soft reset.
	1126	+ *
	1127	+ * Proper error code is unknown here, it would be -EPIPE if device side
	1128	+ * of enadpoit halted (aka STALL), and -EPROTO if not (transaction error)
	1129	+ * We use -EPROTO, if device is stalled it should return a stall error on
	1130	+ * next transfer, which then will return -EPIPE, and device side stall is
	1131	+ * noted and cleared by class driver.
778	1132	*/
779		- list_for_each_entry(cur_td, &ep->cancelled_td_list, cancelled_td_list) {
780		- xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
781		- "Removing canceled TD starting at 0x%llx (dma).",
782		- (unsigned long long)xhci_trb_virt_to_dma(
783		- cur_td->start_seg, cur_td->first_trb));
784		- ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
785		- if (!ep_ring) {
786		- /* This shouldn't happen unless a driver is mucking
787		- * with the stream ID after submission. This will
788		- * leave the TD on the hardware ring, and the hardware
789		- * will try to execute it, and may access a buffer
790		- * that has already been freed. In the best case, the
791		- * hardware will execute it, and the event handler will
792		- * ignore the completion event for that TD, since it was
793		- * removed from the td_list for that endpoint. In
794		- * short, don't muck with the stream ID after
795		- * submission.
796		- */
797		- xhci_warn(xhci, "WARN Cancelled URB %p "
798		- "has invalid stream ID %u.\n",
799		- cur_td->urb,
800		- cur_td->urb->stream_id);
801		- goto remove_finished_td;
802		- }
803		- /*
804		- * If we stopped on the TD we need to cancel, then we have to
805		- * move the xHC endpoint ring dequeue pointer past this TD.
806		- */
807		- hw_deq = xhci_get_hw_deq(xhci, vdev, ep_index,
808		- cur_td->urb->stream_id);
809		- hw_deq &= ~0xf;
	1133	+ switch (GET_EP_CTX_STATE(ep_ctx)) {
	1134	+ case EP_STATE_HALTED:
	1135	+ xhci_dbg(xhci, "Stop ep completion raced with stall, reset ep\n");
	1136	+ if (ep->ep_state & EP_HAS_STREAMS) {
	1137	+ reset_type = EP_SOFT_RESET;
	1138	+ } else {
	1139	+ reset_type = EP_HARD_RESET;
	1140	+ td = find_halted_td(ep);
	1141	+ if (td)
	1142	+ td->status = -EPROTO;
	1143	+ }
	1144	+ /* reset ep, reset handler cleans up cancelled tds */
	1145	+ err = xhci_handle_halted_endpoint(xhci, ep, 0, td,
	1146	+ reset_type);
	1147	+ if (err)
	1148	+ break;
	1149	+ xhci_stop_watchdog_timer_in_irq(xhci, ep);
	1150	+ return;
	1151	+ case EP_STATE_RUNNING:
	1152	+ /* Race, HW handled stop ep cmd before ep was running */
	1153	+ xhci_dbg(xhci, "Stop ep completion ctx error, ep is running\n");
810	1154
811		- if (trb_in_td(xhci, cur_td->start_seg, cur_td->first_trb,
812		- cur_td->last_trb, hw_deq, false)) {
813		- xhci_find_new_dequeue_state(xhci, slot_id, ep_index,
814		- cur_td->urb->stream_id,
815		- cur_td, &deq_state);
816		- } else {
817		- td_to_noop(xhci, ep_ring, cur_td, false);
818		- }
	1155	+ command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
	1156	+ if (!command)
	1157	+ xhci_stop_watchdog_timer_in_irq(xhci, ep);
819	1158
820		-remove_finished_td:
821		- /*
822		- * The event handler won't see a completion for this TD anymore,
823		- * so remove it from the endpoint ring's TD list. Keep it in
824		- * the cancelled TD list for URB completion later.
825		- */
826		- list_del_init(&cur_td->td_list);
	1159	+ mod_timer(&ep->stop_cmd_timer,
	1160	+ jiffies + XHCI_STOP_EP_CMD_TIMEOUT * HZ);
	1161	+ xhci_queue_stop_endpoint(xhci, command, slot_id, ep_index, 0);
	1162	+ xhci_ring_cmd_db(xhci);
	1163	+
	1164	+ return;
	1165	+ default:
	1166	+ break;
	1167	+ }
827	1168	}
828		-
	1169	+ /* will queue a set TR deq if stopped on a cancelled, uncleared TD */
	1170	+ xhci_invalidate_cancelled_tds(ep);
829	1171	xhci_stop_watchdog_timer_in_irq(xhci, ep);
830	1172
831		- /* If necessary, queue a Set Transfer Ring Dequeue Pointer command */
832		- if (deq_state.new_deq_ptr && deq_state.new_deq_seg) {
833		- xhci_queue_new_dequeue_state(xhci, slot_id, ep_index,
834		- &deq_state);
835		- xhci_ring_cmd_db(xhci);
836		- } else {
837		- /* Otherwise ring the doorbell(s) to restart queued transfers */
838		- ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
839		- }
840		-
841		- /*
842		- * Drop the lock and complete the URBs in the cancelled TD list.
843		- * New TDs to be cancelled might be added to the end of the list before
844		- * we can complete all the URBs for the TDs we already unlinked.
845		- * So stop when we've completed the URB for the last TD we unlinked.
846		- */
847		- do {
848		- cur_td = list_first_entry(&ep->cancelled_td_list,
849		- struct xhci_td, cancelled_td_list);
850		- list_del_init(&cur_td->cancelled_td_list);
851		-
852		- /* Clean up the cancelled URB */
853		- /* Doesn't matter what we pass for status, since the core will
854		- * just overwrite it (because the URB has been unlinked).
855		- */
856		- ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
857		- xhci_unmap_td_bounce_buffer(xhci, ep_ring, cur_td);
858		- inc_td_cnt(cur_td->urb);
859		- if (last_td_in_urb(cur_td))
860		- xhci_giveback_urb_in_irq(xhci, cur_td, 0);
861		-
862		- /* Stop processing the cancelled list if the watchdog timer is
863		- * running.
864		- */
865		- if (xhci->xhc_state & XHCI_STATE_DYING)
866		- return;
867		- } while (cur_td != last_unlinked_td);
868		-
869		- /* Return to the event handler with xhci->lock re-acquired */
	1173	+ /* Otherwise ring the doorbell(s) to restart queued transfers */
	1174	+ xhci_giveback_invalidated_tds(ep);
	1175	+ ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
870	1176	}
871	1177
872	1178	static void xhci_kill_ring_urbs(struct xhci_hcd xhci, struct xhci_ring ring)
..	..	@@ -896,7 +1202,10 @@
896	1202	struct xhci_virt_ep *ep;
897	1203	struct xhci_ring *ring;
898	1204
899		- ep = &xhci->devs[slot_id]->eps[ep_index];
	1205	+ ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
	1206	+ if (!ep)
	1207	+ return;
	1208	+
900	1209	if ((ep->ep_state & EP_HAS_STREAMS) \|\|
901	1210	(ep->ep_state & EP_GETTING_NO_STREAMS)) {
902	1211	int stream_id;
..	..	@@ -988,6 +1297,8 @@
988	1297	struct xhci_virt_ep *ep = from_timer(ep, t, stop_cmd_timer);
989	1298	struct xhci_hcd *xhci = ep->xhci;
990	1299	unsigned long flags;
	1300	+ u32 usbsts;
	1301	+ char str[XHCI_MSG_MAX];
991	1302
992	1303	spin_lock_irqsave(&xhci->lock, flags);
993	1304
..	..	@@ -998,8 +1309,11 @@
998	1309	xhci_dbg(xhci, "Stop EP timer raced with cmd completion, exit");
999	1310	return;
1000	1311	}
	1312	+ usbsts = readl(&xhci->op_regs->status);
1001	1313
1002	1314	xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n");
	1315	+ xhci_warn(xhci, "USBSTS:%s\n", xhci_decode_usbsts(str, usbsts));
	1316	+
1003	1317	ep->ep_state &= ~EP_STOP_CMD_PENDING;
1004	1318
1005	1319	xhci_halt(xhci);
..	..	@@ -1022,10 +1336,7 @@
1022	1336	unsigned int ep_index)
1023	1337	{
1024	1338	union xhci_trb *dequeue_temp;
1025		- int num_trbs_free_temp;
1026		- bool revert = false;
1027	1339
1028		- num_trbs_free_temp = ep_ring->num_trbs_free;
1029	1340	dequeue_temp = ep_ring->dequeue;
1030	1341
1031	1342	/* If we get two back-to-back stalls, and the first stalled transfer
..	..	@@ -1040,8 +1351,6 @@
1040	1351	}
1041	1352
1042	1353	while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) {
1043		- /* We have more usable TRBs */
1044		- ep_ring->num_trbs_free++;
1045	1354	ep_ring->dequeue++;
1046	1355	if (trb_is_link(ep_ring->dequeue)) {
1047	1356	if (ep_ring->dequeue ==
..	..	@@ -1051,14 +1360,9 @@
1051	1360	ep_ring->dequeue = ep_ring->deq_seg->trbs;
1052	1361	}
1053	1362	if (ep_ring->dequeue == dequeue_temp) {
1054		- revert = true;
	1363	+ xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
1055	1364	break;
1056	1365	}
1057		- }
1058		-
1059		- if (revert) {
1060		- xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
1061		- ep_ring->num_trbs_free = num_trbs_free_temp;
1062	1366	}
1063	1367	}
1064	1368
..	..	@@ -1075,10 +1379,10 @@
1075	1379	unsigned int ep_index;
1076	1380	unsigned int stream_id;
1077	1381	struct xhci_ring *ep_ring;
1078		- struct xhci_virt_device *dev;
1079	1382	struct xhci_virt_ep *ep;
1080	1383	struct xhci_ep_ctx *ep_ctx;
1081	1384	struct xhci_slot_ctx *slot_ctx;
	1385	+ struct xhci_td td, tmp_td;
1082	1386
1083	1387	ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1084	1388	stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2]));
..	..	@@ -1086,8 +1390,7 @@
1086	1390	if (!ep)
1087	1391	return;
1088	1392
1089		- dev = xhci->devs[slot_id];
1090		- ep_ring = xhci_stream_id_to_ring(dev, ep_index, stream_id);
	1393	+ ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id);
1091	1394	if (!ep_ring) {
1092	1395	xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n",
1093	1396	stream_id);
..	..	@@ -1095,8 +1398,8 @@
1095	1398	goto cleanup;
1096	1399	}
1097	1400
1098		- ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
1099		- slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx);
	1401	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
	1402	+ slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx);
1100	1403	trace_xhci_handle_cmd_set_deq(slot_ctx);
1101	1404	trace_xhci_handle_cmd_set_deq_ep(ep_ctx);
1102	1405
..	..	@@ -1149,7 +1452,7 @@
1149	1452	/* Update the ring's dequeue segment and dequeue pointer
1150	1453	* to reflect the new position.
1151	1454	*/
1152		- update_ring_for_set_deq_completion(xhci, dev,
	1455	+ update_ring_for_set_deq_completion(xhci, ep->vdev,
1153	1456	ep_ring, ep_index);
1154	1457	} else {
1155	1458	xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n");
..	..	@@ -1157,7 +1460,20 @@
1157	1460	ep->queued_deq_seg, ep->queued_deq_ptr);
1158	1461	}
1159	1462	}
1160		-
	1463	+ /* HW cached TDs cleared from cache, give them back */
	1464	+ list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list,
	1465	+ cancelled_td_list) {
	1466	+ ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb);
	1467	+ if (td->cancel_status == TD_CLEARING_CACHE) {
	1468	+ td->cancel_status = TD_CLEARED;
	1469	+ xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n",
	1470	+ __func__, td->urb);
	1471	+ xhci_td_cleanup(ep->xhci, td, ep_ring, td->status);
	1472	+ } else {
	1473	+ xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n",
	1474	+ __func__, td->urb, td->cancel_status);
	1475	+ }
	1476	+ }
1161	1477	cleanup:
1162	1478	ep->ep_state &= ~SET_DEQ_PENDING;
1163	1479	ep->queued_deq_seg = NULL;
..	..	@@ -1169,7 +1485,6 @@
1169	1485	static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
1170	1486	union xhci_trb *trb, u32 cmd_comp_code)
1171	1487	{
1172		- struct xhci_virt_device *vdev;
1173	1488	struct xhci_virt_ep *ep;
1174	1489	struct xhci_ep_ctx *ep_ctx;
1175	1490	unsigned int ep_index;
..	..	@@ -1179,8 +1494,7 @@
1179	1494	if (!ep)
1180	1495	return;
1181	1496
1182		- vdev = xhci->devs[slot_id];
1183		- ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
	1497	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
1184	1498	trace_xhci_handle_cmd_reset_ep(ep_ctx);
1185	1499
1186	1500	/* This command will only fail if the endpoint wasn't halted,
..	..	@@ -1189,27 +1503,19 @@
1189	1503	xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
1190	1504	"Ignoring reset ep completion code of %u", cmd_comp_code);
1191	1505
1192		- /* HW with the reset endpoint quirk needs to have a configure endpoint
1193		- * command complete before the endpoint can be used. Queue that here
1194		- * because the HW can't handle two commands being queued in a row.
1195		- */
1196		- if (xhci->quirks & XHCI_RESET_EP_QUIRK) {
1197		- struct xhci_command *command;
	1506	+ /* Cleanup cancelled TDs as ep is stopped. May queue a Set TR Deq cmd */
	1507	+ xhci_invalidate_cancelled_tds(ep);
1198	1508
1199		- command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1200		- if (!command)
1201		- return;
	1509	+ if (xhci->quirks & XHCI_RESET_EP_QUIRK)
	1510	+ xhci_dbg(xhci, "Note: Removed workaround to queue config ep for this hw");
	1511	+ /* Clear our internal halted state */
	1512	+ ep->ep_state &= ~EP_HALTED;
1202	1513
1203		- xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1204		- "Queueing configure endpoint command");
1205		- xhci_queue_configure_endpoint(xhci, command,
1206		- xhci->devs[slot_id]->in_ctx->dma, slot_id,
1207		- false);
1208		- xhci_ring_cmd_db(xhci);
1209		- } else {
1210		- /* Clear our internal halted state */
1211		- ep->ep_state &= ~EP_HALTED;
1212		- }
	1514	+ xhci_giveback_invalidated_tds(ep);
	1515	+
	1516	+ /* if this was a soft reset, then restart */
	1517	+ if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP)
	1518	+ ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1213	1519	}
1214	1520
1215	1521	static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
..	..	@@ -1239,7 +1545,7 @@
1239	1545	}
1240	1546
1241	1547	static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
1242		- struct xhci_event_cmd *event, u32 cmd_comp_code)
	1548	+ u32 cmd_comp_code)
1243	1549	{
1244	1550	struct xhci_virt_device *virt_dev;
1245	1551	struct xhci_input_control_ctx *ctrl_ctx;
..	..	@@ -1257,6 +1563,8 @@
1257	1563	* is not waiting on the configure endpoint command.
1258	1564	*/
1259	1565	virt_dev = xhci->devs[slot_id];
	1566	+ if (!virt_dev)
	1567	+ return;
1260	1568	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
1261	1569	if (!ctrl_ctx) {
1262	1570	xhci_warn(xhci, "Could not get input context, bad type.\n");
..	..	@@ -1301,24 +1609,27 @@
1301	1609	struct xhci_slot_ctx *slot_ctx;
1302	1610
1303	1611	vdev = xhci->devs[slot_id];
	1612	+ if (!vdev)
	1613	+ return;
1304	1614	slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1305	1615	trace_xhci_handle_cmd_addr_dev(slot_ctx);
1306	1616	}
1307	1617
1308		-static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id,
1309		- struct xhci_event_cmd *event)
	1618	+static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id)
1310	1619	{
1311	1620	struct xhci_virt_device *vdev;
1312	1621	struct xhci_slot_ctx *slot_ctx;
1313	1622
1314	1623	vdev = xhci->devs[slot_id];
	1624	+ if (!vdev) {
	1625	+ xhci_warn(xhci, "Reset device command completion for disabled slot %u\n",
	1626	+ slot_id);
	1627	+ return;
	1628	+ }
1315	1629	slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1316	1630	trace_xhci_handle_cmd_reset_dev(slot_ctx);
1317	1631
1318	1632	xhci_dbg(xhci, "Completed reset device command.\n");
1319		- if (!xhci->devs[slot_id])
1320		- xhci_warn(xhci, "Reset device command completion "
1321		- "for disabled slot %u\n", slot_id);
1322	1633	}
1323	1634
1324	1635	static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
..	..	@@ -1411,13 +1722,18 @@
1411	1722	static void handle_cmd_completion(struct xhci_hcd *xhci,
1412	1723	struct xhci_event_cmd *event)
1413	1724	{
1414		- int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
	1725	+ unsigned int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1415	1726	u64 cmd_dma;
1416	1727	dma_addr_t cmd_dequeue_dma;
1417	1728	u32 cmd_comp_code;
1418	1729	union xhci_trb *cmd_trb;
1419	1730	struct xhci_command *cmd;
1420	1731	u32 cmd_type;
	1732	+
	1733	+ if (slot_id >= MAX_HC_SLOTS) {
	1734	+ xhci_warn(xhci, "Invalid slot_id %u\n", slot_id);
	1735	+ return;
	1736	+ }
1421	1737
1422	1738	cmd_dma = le64_to_cpu(event->cmd_trb);
1423	1739	cmd_trb = xhci->cmd_ring->dequeue;
..	..	@@ -1479,8 +1795,7 @@
1479	1795	break;
1480	1796	case TRB_CONFIG_EP:
1481	1797	if (!cmd->completion)
1482		- xhci_handle_cmd_config_ep(xhci, slot_id, event,
1483		- cmd_comp_code);
	1798	+ xhci_handle_cmd_config_ep(xhci, slot_id, cmd_comp_code);
1484	1799	break;
1485	1800	case TRB_EVAL_CONTEXT:
1486	1801	break;
..	..	@@ -1491,7 +1806,8 @@
1491	1806	WARN_ON(slot_id != TRB_TO_SLOT_ID(
1492	1807	le32_to_cpu(cmd_trb->generic.field[3])));
1493	1808	if (!cmd->completion)
1494		- xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, event);
	1809	+ xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb,
	1810	+ cmd_comp_code);
1495	1811	break;
1496	1812	case TRB_SET_DEQ:
1497	1813	WARN_ON(slot_id != TRB_TO_SLOT_ID(
..	..	@@ -1514,7 +1830,7 @@
1514	1830	*/
1515	1831	slot_id = TRB_TO_SLOT_ID(
1516	1832	le32_to_cpu(cmd_trb->generic.field[3]));
1517		- xhci_handle_cmd_reset_dev(xhci, slot_id, event);
	1833	+ xhci_handle_cmd_reset_dev(xhci, slot_id);
1518	1834	break;
1519	1835	case TRB_NEC_GET_FW:
1520	1836	xhci_handle_cmd_nec_get_fw(xhci, event);
..	..	@@ -1541,11 +1857,8 @@
1541	1857	}
1542	1858
1543	1859	static void handle_vendor_event(struct xhci_hcd *xhci,
1544		- union xhci_trb *event)
	1860	+ union xhci_trb *event, u32 trb_type)
1545	1861	{
1546		- u32 trb_type;
1547		-
1548		- trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->generic.field[3]));
1549	1862	xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
1550	1863	if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
1551	1864	handle_cmd_completion(xhci, &event->event_cmd);
..	..	@@ -1619,18 +1932,19 @@
1619	1932	"WARN: xHC returned failed port status event\n");
1620	1933
1621	1934	port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
1622		- xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id);
1623		-
1624	1935	max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
	1936	+
1625	1937	if ((port_id <= 0) \|\| (port_id > max_ports)) {
1626		- xhci_warn(xhci, "Invalid port id %d\n", port_id);
	1938	+ xhci_warn(xhci, "Port change event with invalid port ID %d\n",
	1939	+ port_id);
1627	1940	inc_deq(xhci, xhci->event_ring);
1628	1941	return;
1629	1942	}
1630	1943
1631	1944	port = &xhci->hw_ports[port_id - 1];
1632	1945	if (!port \|\| !port->rhub \|\| port->hcd_portnum == DUPLICATE_ENTRY) {
1633		- xhci_warn(xhci, "Event for invalid port %u\n", port_id);
	1946	+ xhci_warn(xhci, "Port change event, no port for port ID %u\n",
	1947	+ port_id);
1634	1948	bogus_port_status = true;
1635	1949	goto cleanup;
1636	1950	}
..	..	@@ -1643,9 +1957,12 @@
1643	1957	}
1644	1958
1645	1959	hcd = port->rhub->hcd;
1646		- bus_state = &xhci->bus_state[hcd_index(hcd)];
	1960	+ bus_state = &port->rhub->bus_state;
1647	1961	hcd_portnum = port->hcd_portnum;
1648	1962	portsc = readl(port->addr);
	1963	+
	1964	+ xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n",
	1965	+ hcd->self.busnum, hcd_portnum + 1, port_id, portsc);
1649	1966
1650	1967	trace_xhci_handle_port_status(hcd_portnum, portsc);
1651	1968
..	..	@@ -1678,8 +1995,8 @@
1678	1995	*/
1679	1996	bus_state->port_remote_wakeup \|= 1 << hcd_portnum;
1680	1997	xhci_test_and_clear_bit(xhci, port, PORT_PLC);
1681		- xhci_set_link_state(xhci, port, XDEV_U0);
1682	1998	usb_hcd_start_port_resume(&hcd->self, hcd_portnum);
	1999	+ xhci_set_link_state(xhci, port, XDEV_U0);
1683	2000	/* Need to wait until the next link state change
1684	2001	* indicates the device is actually in U0.
1685	2002	*/
..	..	@@ -1708,6 +2025,7 @@
1708	2025	(portsc & PORT_PLS_MASK) == XDEV_U1 \|\|
1709	2026	(portsc & PORT_PLS_MASK) == XDEV_U2)) {
1710	2027	xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
	2028	+ complete(&bus_state->u3exit_done[hcd_portnum]);
1711	2029	/* We've just brought the device into U0/1/2 through either the
1712	2030	* Resume state after a device remote wakeup, or through the
1713	2031	* U3Exit state after a host-initiated resume. If it's a device
..	..	@@ -1765,7 +2083,8 @@
1765	2083	* bits are still set. When an event occurs, switch over to
1766	2084	* polling to avoid losing status changes.
1767	2085	*/
1768		- xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
	2086	+ xhci_dbg(xhci, "%s: starting usb%d port polling.\n",
	2087	+ __func__, hcd->self.busnum);
1769	2088	set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1770	2089	spin_unlock(&xhci->lock);
1771	2090	/* Pass this up to the core */
..	..	@@ -1840,34 +2159,21 @@
1840	2159	return NULL;
1841	2160	}
1842	2161
1843		-static void xhci_cleanup_halted_endpoint(struct xhci_hcd *xhci,
1844		- unsigned int slot_id, unsigned int ep_index,
1845		- unsigned int stream_id, struct xhci_td *td,
1846		- enum xhci_ep_reset_type reset_type)
	2162	+static void xhci_clear_hub_tt_buffer(struct xhci_hcd xhci, struct xhci_td td,
	2163	+ struct xhci_virt_ep *ep)
1847	2164	{
1848		- struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
1849		- struct xhci_command *command;
1850		-
1851	2165	/*
1852		- * Avoid resetting endpoint if link is inactive. Can cause host hang.
1853		- * Device will be reset soon to recover the link so don't do anything
	2166	+ * As part of low/full-speed endpoint-halt processing
	2167	+ * we must clear the TT buffer (USB 2.0 specification 11.17.5).
1854	2168	*/
1855		- if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR)
1856		- return;
1857		-
1858		- command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1859		- if (!command)
1860		- return;
1861		-
1862		- ep->ep_state \|= EP_HALTED;
1863		-
1864		- xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type);
1865		-
1866		- if (reset_type == EP_HARD_RESET) {
1867		- ep->ep_state \|= EP_HARD_CLEAR_TOGGLE;
1868		- xhci_cleanup_stalled_ring(xhci, ep_index, stream_id, td);
	2169	+ if (td->urb->dev->tt && !usb_pipeint(td->urb->pipe) &&
	2170	+ (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) &&
	2171	+ !(ep->ep_state & EP_CLEARING_TT)) {
	2172	+ ep->ep_state \|= EP_CLEARING_TT;
	2173	+ td->urb->ep->hcpriv = td->urb->dev;
	2174	+ if (usb_hub_clear_tt_buffer(td->urb))
	2175	+ ep->ep_state &= ~EP_CLEARING_TT;
1869	2176	}
1870		- xhci_ring_cmd_db(xhci);
1871	2177	}
1872	2178
1873	2179	/* Check if an error has halted the endpoint ring. The class driver will
..	..	@@ -1910,97 +2216,95 @@
1910	2216	return 0;
1911	2217	}
1912	2218
1913		-static int xhci_td_cleanup(struct xhci_hcd xhci, struct xhci_td td,
1914		- struct xhci_ring ep_ring, int status)
	2219	+static int finish_td(struct xhci_hcd xhci, struct xhci_virt_ep ep,
	2220	+ struct xhci_ring ep_ring, struct xhci_td td,
	2221	+ u32 trb_comp_code)
1915	2222	{
1916		- struct urb *urb = NULL;
1917		-
1918		- /* Clean up the endpoint's TD list */
1919		- urb = td->urb;
1920		-
1921		- /* if a bounce buffer was used to align this td then unmap it */
1922		- xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);
1923		-
1924		- /* Do one last check of the actual transfer length.
1925		- * If the host controller said we transferred more data than the buffer
1926		- * length, urb->actual_length will be a very big number (since it's
1927		- * unsigned). Play it safe and say we didn't transfer anything.
1928		- */
1929		- if (urb->actual_length > urb->transfer_buffer_length) {
1930		- xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
1931		- urb->transfer_buffer_length, urb->actual_length);
1932		- urb->actual_length = 0;
1933		- *status = 0;
1934		- }
1935		- list_del_init(&td->td_list);
1936		- /* Was this TD slated to be cancelled but completed anyway? */
1937		- if (!list_empty(&td->cancelled_td_list))
1938		- list_del_init(&td->cancelled_td_list);
1939		-
1940		- inc_td_cnt(urb);
1941		- /* Giveback the urb when all the tds are completed */
1942		- if (last_td_in_urb(td)) {
1943		- if ((urb->actual_length != urb->transfer_buffer_length &&
1944		- (urb->transfer_flags & URB_SHORT_NOT_OK)) \|\|
1945		- (*status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc)))
1946		- xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
1947		- urb, urb->actual_length,
1948		- urb->transfer_buffer_length, *status);
1949		-
1950		- /* set isoc urb status to 0 just as EHCI, UHCI, and OHCI */
1951		- if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
1952		- *status = 0;
1953		- xhci_giveback_urb_in_irq(xhci, td, *status);
1954		- }
1955		-
1956		- return 0;
1957		-}
1958		-
1959		-static int finish_td(struct xhci_hcd xhci, struct xhci_td td,
1960		- struct xhci_transfer_event *event,
1961		- struct xhci_virt_ep ep, int status)
1962		-{
1963		- struct xhci_virt_device *xdev;
1964	2223	struct xhci_ep_ctx *ep_ctx;
1965		- struct xhci_ring *ep_ring;
1966		- unsigned int slot_id;
1967		- u32 trb_comp_code;
1968		- int ep_index;
	2224	+ int trbs_freed;
1969	2225
1970		- slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1971		- xdev = xhci->devs[slot_id];
1972		- ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1973		- ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1974		- ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1975		- trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
	2226	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index);
1976	2227
1977		- if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID \|\|
1978		- trb_comp_code == COMP_STOPPED \|\|
1979		- trb_comp_code == COMP_STOPPED_SHORT_PACKET) {
1980		- /* The Endpoint Stop Command completion will take care of any
1981		- * stopped TDs. A stopped TD may be restarted, so don't update
	2228	+ switch (trb_comp_code) {
	2229	+ case COMP_STOPPED_LENGTH_INVALID:
	2230	+ case COMP_STOPPED_SHORT_PACKET:
	2231	+ case COMP_STOPPED:
	2232	+ /*
	2233	+ * The "Stop Endpoint" completion will take care of any
	2234	+ * stopped TDs. A stopped TD may be restarted, so don't update
1982	2235	* the ring dequeue pointer or take this TD off any lists yet.
1983	2236	*/
1984	2237	return 0;
1985		- }
1986		- if (trb_comp_code == COMP_STALL_ERROR \|\|
1987		- xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
1988		- trb_comp_code)) {
1989		- /* Issue a reset endpoint command to clear the host side
1990		- * halt, followed by a set dequeue command to move the
1991		- * dequeue pointer past the TD.
1992		- * The class driver clears the device side halt later.
	2238	+ case COMP_USB_TRANSACTION_ERROR:
	2239	+ case COMP_BABBLE_DETECTED_ERROR:
	2240	+ case COMP_SPLIT_TRANSACTION_ERROR:
	2241	+ /*
	2242	+ * If endpoint context state is not halted we might be
	2243	+ * racing with a reset endpoint command issued by a unsuccessful
	2244	+ * stop endpoint completion (context error). In that case the
	2245	+ * td should be on the cancelled list, and EP_HALTED flag set.
	2246	+ *
	2247	+ * Or then it's not halted due to the 0.95 spec stating that a
	2248	+ * babbling control endpoint should not halt. The 0.96 spec
	2249	+ * again says it should. Some HW claims to be 0.95 compliant,
	2250	+ * but it halts the control endpoint anyway.
1993	2251	*/
1994		- xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
1995		- ep_ring->stream_id, td, EP_HARD_RESET);
1996		- } else {
1997		- /* Update ring dequeue pointer */
1998		- while (ep_ring->dequeue != td->last_trb)
1999		- inc_deq(xhci, ep_ring);
2000		- inc_deq(xhci, ep_ring);
	2252	+ if (GET_EP_CTX_STATE(ep_ctx) != EP_STATE_HALTED) {
	2253	+ /*
	2254	+ * If EP_HALTED is set and TD is on the cancelled list
	2255	+ * the TD and dequeue pointer will be handled by reset
	2256	+ * ep command completion
	2257	+ */
	2258	+ if ((ep->ep_state & EP_HALTED) &&
	2259	+ !list_empty(&td->cancelled_td_list)) {
	2260	+ xhci_dbg(xhci, "Already resolving halted ep for 0x%llx\n",
	2261	+ (unsigned long long)xhci_trb_virt_to_dma(
	2262	+ td->start_seg, td->first_trb));
	2263	+ return 0;
	2264	+ }
	2265	+ /* endpoint not halted, don't reset it */
	2266	+ break;
	2267	+ }
	2268	+ /* Almost same procedure as for STALL_ERROR below */
	2269	+ xhci_clear_hub_tt_buffer(xhci, td, ep);
	2270	+ xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,
	2271	+ EP_HARD_RESET);
	2272	+ return 0;
	2273	+ case COMP_STALL_ERROR:
	2274	+ /*
	2275	+ * xhci internal endpoint state will go to a "halt" state for
	2276	+ * any stall, including default control pipe protocol stall.
	2277	+ * To clear the host side halt we need to issue a reset endpoint
	2278	+ * command, followed by a set dequeue command to move past the
	2279	+ * TD.
	2280	+ * Class drivers clear the device side halt from a functional
	2281	+ * stall later. Hub TT buffer should only be cleared for FS/LS
	2282	+ * devices behind HS hubs for functional stalls.
	2283	+ */
	2284	+ if (ep->ep_index != 0)
	2285	+ xhci_clear_hub_tt_buffer(xhci, td, ep);
	2286	+
	2287	+ xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,
	2288	+ EP_HARD_RESET);
	2289	+
	2290	+ return 0; /* xhci_handle_halted_endpoint marked td cancelled */
	2291	+ default:
	2292	+ break;
2001	2293	}
2002	2294
2003		- return xhci_td_cleanup(xhci, td, ep_ring, status);
	2295	+ /* Update ring dequeue pointer */
	2296	+ trbs_freed = xhci_num_trbs_to(ep_ring->deq_seg, ep_ring->dequeue,
	2297	+ td->last_trb_seg, td->last_trb,
	2298	+ ep_ring->num_segs);
	2299	+ if (trbs_freed < 0)
	2300	+ xhci_dbg(xhci, "Failed to count freed trbs at TD finish\n");
	2301	+ else
	2302	+ ep_ring->num_trbs_free += trbs_freed;
	2303	+ ep_ring->dequeue = td->last_trb;
	2304	+ ep_ring->deq_seg = td->last_trb_seg;
	2305	+ inc_deq(xhci, ep_ring);
	2306	+
	2307	+ return xhci_td_cleanup(xhci, td, ep_ring, td->status);
2004	2308	}
2005	2309
2006	2310	/* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */
..	..	@@ -2021,23 +2325,17 @@
2021	2325	/*
2022	2326	* Process control tds, update urb status and actual_length.
2023	2327	*/
2024		-static int process_ctrl_td(struct xhci_hcd xhci, struct xhci_td td,
2025		- union xhci_trb ep_trb, struct xhci_transfer_event event,
2026		- struct xhci_virt_ep ep, int status)
	2328	+static int process_ctrl_td(struct xhci_hcd xhci, struct xhci_virt_ep ep,
	2329	+ struct xhci_ring ep_ring, struct xhci_td td,
	2330	+ union xhci_trb ep_trb, struct xhci_transfer_event event)
2027	2331	{
2028		- struct xhci_virt_device *xdev;
2029		- unsigned int slot_id;
2030		- int ep_index;
2031	2332	struct xhci_ep_ctx *ep_ctx;
2032	2333	u32 trb_comp_code;
2033	2334	u32 remaining, requested;
2034	2335	u32 trb_type;
2035	2336
2036	2337	trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3]));
2037		- slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
2038		- xdev = xhci->devs[slot_id];
2039		- ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
2040		- ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
	2338	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index);
2041	2339	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2042	2340	requested = td->urb->transfer_buffer_length;
2043	2341	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
..	..	@@ -2047,13 +2345,13 @@
2047	2345	if (trb_type != TRB_STATUS) {
2048	2346	xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n",
2049	2347	(trb_type == TRB_DATA) ? "data" : "setup");
2050		- *status = -ESHUTDOWN;
	2348	+ td->status = -ESHUTDOWN;
2051	2349	break;
2052	2350	}
2053		- *status = 0;
	2351	+ td->status = 0;
2054	2352	break;
2055	2353	case COMP_SHORT_PACKET:
2056		- *status = 0;
	2354	+ td->status = 0;
2057	2355	break;
2058	2356	case COMP_STOPPED_SHORT_PACKET:
2059	2357	if (trb_type == TRB_DATA \|\| trb_type == TRB_NORMAL)
..	..	@@ -2085,8 +2383,8 @@
2085	2383	ep_ctx, trb_comp_code))
2086	2384	break;
2087	2385	xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n",
2088		- trb_comp_code, ep_index);
2089		- /* else fall through */
	2386	+ trb_comp_code, ep->ep_index);
	2387	+ fallthrough;
2090	2388	case COMP_STALL_ERROR:
2091	2389	/* Did we transfer part of the data (middle) phase? */
2092	2390	if (trb_type == TRB_DATA \|\| trb_type == TRB_NORMAL)
..	..	@@ -2117,17 +2415,16 @@
2117	2415	td->urb->actual_length = requested;
2118	2416
2119	2417	finish_td:
2120		- return finish_td(xhci, td, event, ep, status);
	2418	+ return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2121	2419	}
2122	2420
2123	2421	/*
2124	2422	* Process isochronous tds, update urb packet status and actual_length.
2125	2423	*/
2126		-static int process_isoc_td(struct xhci_hcd xhci, struct xhci_td td,
2127		- union xhci_trb ep_trb, struct xhci_transfer_event event,
2128		- struct xhci_virt_ep ep, int status)
	2424	+static int process_isoc_td(struct xhci_hcd xhci, struct xhci_virt_ep ep,
	2425	+ struct xhci_ring ep_ring, struct xhci_td td,
	2426	+ union xhci_trb ep_trb, struct xhci_transfer_event event)
2129	2427	{
2130		- struct xhci_ring *ep_ring;
2131	2428	struct urb_priv *urb_priv;
2132	2429	int idx;
2133	2430	struct usb_iso_packet_descriptor *frame;
..	..	@@ -2136,7 +2433,6 @@
2136	2433	u32 remaining, requested, ep_trb_len;
2137	2434	int short_framestatus;
2138	2435
2139		- ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2140	2436	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2141	2437	urb_priv = td->urb->hcpriv;
2142	2438	idx = urb_priv->num_tds_done;
..	..	@@ -2197,26 +2493,23 @@
2197	2493	}
2198	2494
2199	2495	if (sum_trbs_for_length)
2200		- frame->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb) +
	2496	+ frame->actual_length = sum_trb_lengths(xhci, ep->ring, ep_trb) +
2201	2497	ep_trb_len - remaining;
2202	2498	else
2203	2499	frame->actual_length = requested;
2204	2500
2205	2501	td->urb->actual_length += frame->actual_length;
2206	2502
2207		- return finish_td(xhci, td, event, ep, status);
	2503	+ return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2208	2504	}
2209	2505
2210	2506	static int skip_isoc_td(struct xhci_hcd xhci, struct xhci_td td,
2211		- struct xhci_transfer_event *event,
2212		- struct xhci_virt_ep ep, int status)
	2507	+ struct xhci_virt_ep *ep, int status)
2213	2508	{
2214		- struct xhci_ring *ep_ring;
2215	2509	struct urb_priv *urb_priv;
2216	2510	struct usb_iso_packet_descriptor *frame;
2217	2511	int idx;
2218	2512
2219		- ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2220	2513	urb_priv = td->urb->hcpriv;
2221	2514	idx = urb_priv->num_tds_done;
2222	2515	frame = &td->urb->iso_frame_desc[idx];
..	..	@@ -2228,25 +2521,26 @@
2228	2521	frame->actual_length = 0;
2229	2522
2230	2523	/* Update ring dequeue pointer */
2231		- while (ep_ring->dequeue != td->last_trb)
2232		- inc_deq(xhci, ep_ring);
2233		- inc_deq(xhci, ep_ring);
	2524	+ ep->ring->dequeue = td->last_trb;
	2525	+ ep->ring->deq_seg = td->last_trb_seg;
	2526	+ ep->ring->num_trbs_free += td->num_trbs - 1;
	2527	+ inc_deq(xhci, ep->ring);
2234	2528
2235		- return xhci_td_cleanup(xhci, td, ep_ring, status);
	2529	+ return xhci_td_cleanup(xhci, td, ep->ring, status);
2236	2530	}
2237	2531
2238	2532	/*
2239	2533	* Process bulk and interrupt tds, update urb status and actual_length.
2240	2534	*/
2241		-static int process_bulk_intr_td(struct xhci_hcd xhci, struct xhci_td td,
2242		- union xhci_trb ep_trb, struct xhci_transfer_event event,
2243		- struct xhci_virt_ep ep, int status)
	2535	+static int process_bulk_intr_td(struct xhci_hcd xhci, struct xhci_virt_ep ep,
	2536	+ struct xhci_ring ep_ring, struct xhci_td td,
	2537	+ union xhci_trb ep_trb, struct xhci_transfer_event event)
2244	2538	{
2245		- struct xhci_ring *ep_ring;
	2539	+ struct xhci_slot_ctx *slot_ctx;
2246	2540	u32 trb_comp_code;
2247	2541	u32 remaining, requested, ep_trb_len;
2248	2542
2249		- ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
	2543	+ slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx);
2250	2544	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2251	2545	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2252	2546	ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
..	..	@@ -2254,6 +2548,7 @@
2254	2548
2255	2549	switch (trb_comp_code) {
2256	2550	case COMP_SUCCESS:
	2551	+ ep_ring->err_count = 0;
2257	2552	/* handle success with untransferred data as short packet */
2258	2553	if (ep_trb != td->last_trb \|\| remaining) {
2259	2554	xhci_warn(xhci, "WARN Successful completion on short TX\n");
..	..	@@ -2261,13 +2556,13 @@
2261	2556	td->urb->ep->desc.bEndpointAddress,
2262	2557	requested, remaining);
2263	2558	}
2264		- *status = 0;
	2559	+ td->status = 0;
2265	2560	break;
2266	2561	case COMP_SHORT_PACKET:
2267	2562	xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2268	2563	td->urb->ep->desc.bEndpointAddress,
2269	2564	requested, remaining);
2270		- *status = 0;
	2565	+ td->status = 0;
2271	2566	break;
2272	2567	case COMP_STOPPED_SHORT_PACKET:
2273	2568	td->urb->actual_length = remaining;
..	..	@@ -2277,6 +2572,17 @@
2277	2572	ep_trb_len = 0;
2278	2573	remaining = 0;
2279	2574	break;
	2575	+ case COMP_USB_TRANSACTION_ERROR:
	2576	+ if (xhci->quirks & XHCI_NO_SOFT_RETRY \|\|
	2577	+ (ep_ring->err_count++ > MAX_SOFT_RETRY) \|\|
	2578	+ le32_to_cpu(slot_ctx->tt_info) & TT_SLOT)
	2579	+ break;
	2580	+
	2581	+ td->status = 0;
	2582	+
	2583	+ xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,
	2584	+ EP_SOFT_RESET);
	2585	+ return 0;
2280	2586	default:
2281	2587	/* do nothing */
2282	2588	break;
..	..	@@ -2294,7 +2600,8 @@
2294	2600	remaining);
2295	2601	td->urb->actual_length = 0;
2296	2602	}
2297		- return finish_td(xhci, td, event, ep, status);
	2603	+
	2604	+ return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2298	2605	}
2299	2606
2300	2607	/*
..	..	@@ -2305,7 +2612,6 @@
2305	2612	static int handle_tx_event(struct xhci_hcd *xhci,
2306	2613	struct xhci_transfer_event *event)
2307	2614	{
2308		- struct xhci_virt_device *xdev;
2309	2615	struct xhci_virt_ep *ep;
2310	2616	struct xhci_ring *ep_ring;
2311	2617	unsigned int slot_id;
..	..	@@ -2332,9 +2638,8 @@
2332	2638	goto err_out;
2333	2639	}
2334	2640
2335		- xdev = xhci->devs[slot_id];
2336	2641	ep_ring = xhci_dma_to_transfer_ring(ep, ep_trb_dma);
2337		- ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
	2642	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
2338	2643
2339	2644	if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) {
2340	2645	xhci_err(xhci,
..	..	@@ -2350,8 +2655,8 @@
2350	2655	case COMP_USB_TRANSACTION_ERROR:
2351	2656	case COMP_INVALID_STREAM_TYPE_ERROR:
2352	2657	case COMP_INVALID_STREAM_ID_ERROR:
2353		- xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index, 0,
2354		- NULL, EP_SOFT_RESET);
	2658	+ xhci_handle_halted_endpoint(xhci, ep, 0, NULL,
	2659	+ EP_SOFT_RESET);
2355	2660	goto cleanup;
2356	2661	case COMP_RING_UNDERRUN:
2357	2662	case COMP_RING_OVERRUN:
..	..	@@ -2406,10 +2711,13 @@
2406	2711	case COMP_STALL_ERROR:
2407	2712	xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id,
2408	2713	ep_index);
2409		- ep->ep_state \|= EP_HALTED;
2410	2714	status = -EPIPE;
2411	2715	break;
2412	2716	case COMP_SPLIT_TRANSACTION_ERROR:
	2717	+ xhci_dbg(xhci, "Split transaction error for slot %u ep %u\n",
	2718	+ slot_id, ep_index);
	2719	+ status = -EPROTO;
	2720	+ break;
2413	2721	case COMP_USB_TRANSACTION_ERROR:
2414	2722	xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n",
2415	2723	slot_id, ep_index);
..	..	@@ -2527,6 +2835,14 @@
2527	2835	xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n",
2528	2836	slot_id, ep_index);
2529	2837	}
	2838	+ if (trb_comp_code == COMP_STALL_ERROR \|\|
	2839	+ xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
	2840	+ trb_comp_code)) {
	2841	+ xhci_handle_halted_endpoint(xhci, ep,
	2842	+ ep_ring->stream_id,
	2843	+ NULL,
	2844	+ EP_HARD_RESET);
	2845	+ }
2530	2846	goto cleanup;
2531	2847	}
2532	2848
..	..	@@ -2584,7 +2900,7 @@
2584	2900	return -ESHUTDOWN;
2585	2901	}
2586	2902
2587		- skip_isoc_td(xhci, td, event, ep, &status);
	2903	+ skip_isoc_td(xhci, td, ep, status);
2588	2904	goto cleanup;
2589	2905	}
2590	2906	if (trb_comp_code == COMP_SHORT_PACKET)
..	..	@@ -2612,25 +2928,26 @@
2612	2928	* endpoint. Otherwise, the endpoint remains stalled
2613	2929	* indefinitely.
2614	2930	*/
	2931	+
2615	2932	if (trb_is_noop(ep_trb)) {
2616	2933	if (trb_comp_code == COMP_STALL_ERROR \|\|
2617	2934	xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
2618	2935	trb_comp_code))
2619		- xhci_cleanup_halted_endpoint(xhci, slot_id,
2620		- ep_index,
2621		- ep_ring->stream_id,
2622		- td, EP_HARD_RESET);
	2936	+ xhci_handle_halted_endpoint(xhci, ep,
	2937	+ ep_ring->stream_id,
	2938	+ td, EP_HARD_RESET);
2623	2939	goto cleanup;
2624	2940	}
2625	2941
	2942	+ td->status = status;
	2943	+
2626	2944	/* update the urb's actual_length and give back to the core */
2627	2945	if (usb_endpoint_xfer_control(&td->urb->ep->desc))
2628		- process_ctrl_td(xhci, td, ep_trb, event, ep, &status);
	2946	+ process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event);
2629	2947	else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc))
2630		- process_isoc_td(xhci, td, ep_trb, event, ep, &status);
	2948	+ process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event);
2631	2949	else
2632		- process_bulk_intr_td(xhci, td, ep_trb, event, ep,
2633		- &status);
	2950	+ process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event);
2634	2951	cleanup:
2635	2952	handling_skipped_tds = ep->skip &&
2636	2953	trb_comp_code != COMP_MISSED_SERVICE_ERROR &&
..	..	@@ -2671,10 +2988,11 @@
2671	2988	* Returns >0 for "possibly more events to process" (caller should call again),
2672	2989	* otherwise 0 if done. In future, <0 returns should indicate error code.
2673	2990	*/
2674		-static int xhci_handle_event(struct xhci_hcd *xhci)
	2991	+int xhci_handle_event(struct xhci_hcd *xhci)
2675	2992	{
2676	2993	union xhci_trb *event;
2677	2994	int update_ptrs = 1;
	2995	+ u32 trb_type;
2678	2996	int ret;
2679	2997
2680	2998	/* Event ring hasn't been allocated yet. */
..	..	@@ -2696,31 +3014,30 @@
2696	3014	* speculative reads of the event's flags/data below.
2697	3015	*/
2698	3016	rmb();
	3017	+ trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->event_cmd.flags));
2699	3018	/* FIXME: Handle more event types. */
2700		- switch (le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) {
2701		- case TRB_TYPE(TRB_COMPLETION):
	3019	+
	3020	+ switch (trb_type) {
	3021	+ case TRB_COMPLETION:
2702	3022	handle_cmd_completion(xhci, &event->event_cmd);
2703	3023	break;
2704		- case TRB_TYPE(TRB_PORT_STATUS):
	3024	+ case TRB_PORT_STATUS:
2705	3025	handle_port_status(xhci, event);
2706	3026	update_ptrs = 0;
2707	3027	break;
2708		- case TRB_TYPE(TRB_TRANSFER):
	3028	+ case TRB_TRANSFER:
2709	3029	ret = handle_tx_event(xhci, &event->trans_event);
2710	3030	if (ret >= 0)
2711	3031	update_ptrs = 0;
2712	3032	break;
2713		- case TRB_TYPE(TRB_DEV_NOTE):
	3033	+ case TRB_DEV_NOTE:
2714	3034	handle_device_notification(xhci, event);
2715	3035	break;
2716	3036	default:
2717		- if ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) >=
2718		- TRB_TYPE(48))
2719		- handle_vendor_event(xhci, event);
	3037	+ if (trb_type >= TRB_VENDOR_DEFINED_LOW)
	3038	+ handle_vendor_event(xhci, event, trb_type);
2720	3039	else
2721		- xhci_warn(xhci, "ERROR unknown event type %d\n",
2722		- TRB_FIELD_TO_TYPE(
2723		- le32_to_cpu(event->event_cmd.flags)));
	3040	+ xhci_warn(xhci, "ERROR unknown event type %d\n", trb_type);
2724	3041	}
2725	3042	/* Any of the above functions may drop and re-acquire the lock, so check
2726	3043	* to make sure a watchdog timer didn't mark the host as non-responsive.
..	..	@@ -2740,13 +3057,14 @@
2740	3057	*/
2741	3058	return 1;
2742	3059	}
	3060	+EXPORT_SYMBOL_GPL(xhci_handle_event);
2743	3061
2744	3062	/*
2745	3063	* Update Event Ring Dequeue Pointer:
2746	3064	* - When all events have finished
2747	3065	* - To avoid "Event Ring Full Error" condition
2748	3066	*/
2749		-static void xhci_update_erst_dequeue(struct xhci_hcd *xhci,
	3067	+void xhci_update_erst_dequeue(struct xhci_hcd *xhci,
2750	3068	union xhci_trb *event_ring_deq)
2751	3069	{
2752	3070	u64 temp_64;
..	..	@@ -2775,6 +3093,16 @@
2775	3093	/* Clear the event handler busy flag (RW1C) */
2776	3094	temp_64 \|= ERST_EHB;
2777	3095	xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue);
	3096	+}
	3097	+EXPORT_SYMBOL_GPL(xhci_update_erst_dequeue);
	3098	+
	3099	+static irqreturn_t xhci_vendor_queue_irq_work(struct xhci_hcd *xhci)
	3100	+{
	3101	+ struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci);
	3102	+
	3103	+ if (ops && ops->queue_irq_work)
	3104	+ return ops->queue_irq_work(xhci);
	3105	+ return IRQ_NONE;
2778	3106	}
2779	3107
2780	3108	/*
..	..	@@ -2810,6 +3138,10 @@
2810	3138	ret = IRQ_HANDLED;
2811	3139	goto out;
2812	3140	}
	3141	+
	3142	+ ret = xhci_vendor_queue_irq_work(xhci);
	3143	+ if (ret == IRQ_HANDLED)
	3144	+ goto out;
2813	3145
2814	3146	/*
2815	3147	* Clear the op reg interrupt status first,
..	..	@@ -2848,6 +3180,8 @@
2848	3180	if (event_loop++ < TRBS_PER_SEGMENT / 2)
2849	3181	continue;
2850	3182	xhci_update_erst_dequeue(xhci, event_ring_deq);
	3183	+ event_ring_deq = xhci->event_ring->dequeue;
	3184	+
2851	3185	event_loop = 0;
2852	3186	}
2853	3187
..	..	@@ -2901,6 +3235,7 @@
2901	3235	u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
2902	3236	{
2903	3237	unsigned int num_trbs_needed;
	3238	+ unsigned int link_trb_count = 0;
2904	3239
2905	3240	/* Make sure the endpoint has been added to xHC schedule */
2906	3241	switch (ep_state) {
..	..	@@ -2971,7 +3306,19 @@
2971	3306
2972	3307	ep_ring->enq_seg = ep_ring->enq_seg->next;
2973	3308	ep_ring->enqueue = ep_ring->enq_seg->trbs;
	3309	+
	3310	+ /* prevent infinite loop if all first trbs are link trbs */
	3311	+ if (link_trb_count++ > ep_ring->num_segs) {
	3312	+ xhci_warn(xhci, "Ring is an endless link TRB loop\n");
	3313	+ return -EINVAL;
	3314	+ }
2974	3315	}
	3316	+
	3317	+ if (last_trb_on_seg(ep_ring->enq_seg, ep_ring->enqueue)) {
	3318	+ xhci_warn(xhci, "Missing link TRB at end of ring segment\n");
	3319	+ return -EINVAL;
	3320	+ }
	3321	+
2975	3322	return 0;
2976	3323	}
2977	3324
..	..	@@ -2990,7 +3337,8 @@
2990	3337	struct xhci_ring *ep_ring;
2991	3338	struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2992	3339
2993		- ep_ring = xhci_stream_id_to_ring(xdev, ep_index, stream_id);
	3340	+ ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index,
	3341	+ stream_id);
2994	3342	if (!ep_ring) {
2995	3343	xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n",
2996	3344	stream_id);
..	..	@@ -3272,7 +3620,6 @@
3272	3620	bool more_trbs_coming = true;
3273	3621	bool need_zero_pkt = false;
3274	3622	bool first_trb = true;
3275		- bool en_trb_ent = true;
3276	3623	unsigned int num_trbs;
3277	3624	unsigned int start_cycle, num_sgs = 0;
3278	3625	unsigned int enqd_len, block_len, trb_buff_len, full_len;
..	..	@@ -3309,13 +3656,6 @@
3309	3656	if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1)
3310	3657	need_zero_pkt = true;
3311	3658
3312		- /*
3313		- * Don't enable the ENT flag in the TRB if
3314		- * the EP support bulk streaming protocol.
3315		- */
3316		- if (urb->stream_id)
3317		- en_trb_ent = false;
3318		-
3319	3659	td = &urb_priv->td[0];
3320	3660
3321	3661	/*
..	..	@@ -3347,22 +3687,11 @@
3347	3687	} else
3348	3688	field \|= ring->cycle_state;
3349	3689
3350		- /*
3351		- * Don't enable the ENT flag in the TRB if the
3352		- * transfer length of the TRB isn't an integer
3353		- * multiple of the EP maxpacket.
3354		- */
3355		- if (en_trb_ent &&
3356		- (trb_buff_len % usb_endpoint_maxp(&urb->ep->desc)))
3357		- en_trb_ent = false;
3358		-
3359	3690	/* Chain all the TRBs together; clear the chain bit in the last
3360	3691	* TRB to indicate it's the last TRB in the chain.
3361	3692	*/
3362	3693	if (enqd_len + trb_buff_len < full_len) {
3363	3694	field \|= TRB_CHAIN;
3364		- if (xhci->quirks & XHCI_TRB_ENT_QUIRK && en_trb_ent)
3365		- field \|= TRB_ENT;
3366	3695	if (trb_is_link(ring->enqueue + 1)) {
3367	3696	if (xhci_align_td(xhci, urb, enqd_len,
3368	3697	&trb_buff_len,
..	..	@@ -3378,6 +3707,13 @@
3378	3707	field \|= TRB_IOC;
3379	3708	more_trbs_coming = false;
3380	3709	td->last_trb = ring->enqueue;
	3710	+ td->last_trb_seg = ring->enq_seg;
	3711	+ if (xhci_urb_suitable_for_idt(urb)) {
	3712	+ memcpy(&send_addr, urb->transfer_buffer,
	3713	+ trb_buff_len);
	3714	+ le64_to_cpus(&send_addr);
	3715	+ field \|= TRB_IDT;
	3716	+ }
3381	3717	}
3382	3718
3383	3719	/* Only set interrupt on short packet for IN endpoints */
..	..	@@ -3397,7 +3733,7 @@
3397	3733	upper_32_bits(send_addr),
3398	3734	length_field,
3399	3735	field);
3400		-
	3736	+ td->num_trbs++;
3401	3737	addr += trb_buff_len;
3402	3738	sent_len = trb_buff_len;
3403	3739
..	..	@@ -3421,8 +3757,10 @@
3421	3757	ep_index, urb->stream_id,
3422	3758	1, urb, 1, mem_flags);
3423	3759	urb_priv->td[1].last_trb = ring->enqueue;
	3760	+ urb_priv->td[1].last_trb_seg = ring->enq_seg;
3424	3761	field = TRB_TYPE(TRB_NORMAL) \| ring->cycle_state \| TRB_IOC;
3425	3762	queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field);
	3763	+ urb_priv->td[1].num_trbs++;
3426	3764	}
3427	3765
3428	3766	check_trb_math(urb, enqd_len);
..	..	@@ -3473,6 +3811,7 @@
3473	3811
3474	3812	urb_priv = urb->hcpriv;
3475	3813	td = &urb_priv->td[0];
	3814	+ td->num_trbs = num_trbs;
3476	3815
3477	3816	/*
3478	3817	* Don't give the first TRB to the hardware (by toggling the cycle bit)
..	..	@@ -3516,6 +3855,16 @@
3516	3855
3517	3856	if (urb->transfer_buffer_length > 0) {
3518	3857	u32 length_field, remainder;
	3858	+ u64 addr;
	3859	+
	3860	+ if (xhci_urb_suitable_for_idt(urb)) {
	3861	+ memcpy(&addr, urb->transfer_buffer,
	3862	+ urb->transfer_buffer_length);
	3863	+ le64_to_cpus(&addr);
	3864	+ field \|= TRB_IDT;
	3865	+ } else {
	3866	+ addr = (u64) urb->transfer_dma;
	3867	+ }
3519	3868
3520	3869	remainder = xhci_td_remainder(xhci, 0,
3521	3870	urb->transfer_buffer_length,
..	..	@@ -3527,14 +3876,15 @@
3527	3876	if (setup->bRequestType & USB_DIR_IN)
3528	3877	field \|= TRB_DIR_IN;
3529	3878	queue_trb(xhci, ep_ring, true,
3530		- lower_32_bits(urb->transfer_dma),
3531		- upper_32_bits(urb->transfer_dma),
	3879	+ lower_32_bits(addr),
	3880	+ upper_32_bits(addr),
3532	3881	length_field,
3533	3882	field \| ep_ring->cycle_state);
3534	3883	}
3535	3884
3536	3885	/* Save the DMA address of the last TRB in the TD */
3537	3886	td->last_trb = ep_ring->enqueue;
	3887	+ td->last_trb_seg = ep_ring->enq_seg;
3538	3888
3539	3889	/* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */
3540	3890	/* If the device sent data, the status stage is an OUT transfer */
..	..	@@ -3698,6 +4048,24 @@
3698	4048	return start_frame;
3699	4049	}
3700	4050
	4051	+/* Check if we should generate event interrupt for a TD in an isoc URB */
	4052	+static bool trb_block_event_intr(struct xhci_hcd *xhci, int num_tds, int i)
	4053	+{
	4054	+ if (xhci->hci_version < 0x100)
	4055	+ return false;
	4056	+ /* always generate an event interrupt for the last TD */
	4057	+ if (i == num_tds - 1)
	4058	+ return false;
	4059	+ /*
	4060	+ * If AVOID_BEI is set the host handles full event rings poorly,
	4061	+ * generate an event at least every 8th TD to clear the event ring
	4062	+ */
	4063	+ if (i && xhci->quirks & XHCI_AVOID_BEI)
	4064	+ return !!(i % 8);
	4065	+
	4066	+ return true;
	4067	+}
	4068	+
3701	4069	/* This is for isoc transfer */
3702	4070	static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3703	4071	struct urb *urb, int slot_id, unsigned int ep_index)
..	..	@@ -3761,7 +4129,7 @@
3761	4129	goto cleanup;
3762	4130	}
3763	4131	td = &urb_priv->td[i];
3764		-
	4132	+ td->num_trbs = trbs_per_td;
3765	4133	/* use SIA as default, if frame id is used overwrite it */
3766	4134	sia_frame_id = TRB_SIA;
3767	4135	if (!(urb->transfer_flags & URB_ISO_ASAP) &&
..	..	@@ -3804,11 +4172,9 @@
3804	4172	} else {
3805	4173	more_trbs_coming = false;
3806	4174	td->last_trb = ep_ring->enqueue;
	4175	+ td->last_trb_seg = ep_ring->enq_seg;
3807	4176	field \|= TRB_IOC;
3808		- /* set BEI, except for the last TD */
3809		- if (xhci->hci_version >= 0x100 &&
3810		- !(xhci->quirks & XHCI_AVOID_BEI) &&
3811		- i < num_tds - 1)
	4177	+ if (trb_block_event_intr(xhci, num_tds, i))
3812	4178	field \|= TRB_BEI;
3813	4179	}
3814	4180	/* Calculate TRB length */
..	..	@@ -4089,71 +4455,7 @@
4089	4455	return queue_command(xhci, cmd, 0, 0, 0,
4090	4456	trb_slot_id \| trb_ep_index \| type \| trb_suspend, false);
4091	4457	}
4092		-
4093		-/* Set Transfer Ring Dequeue Pointer command */
4094		-void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci,
4095		- unsigned int slot_id, unsigned int ep_index,
4096		- struct xhci_dequeue_state *deq_state)
4097		-{
4098		- dma_addr_t addr;
4099		- u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
4100		- u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
4101		- u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id);
4102		- u32 trb_sct = 0;
4103		- u32 type = TRB_TYPE(TRB_SET_DEQ);
4104		- struct xhci_virt_ep *ep;
4105		- struct xhci_command *cmd;
4106		- int ret;
4107		-
4108		- xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
4109		- "Set TR Deq Ptr cmd, new deq seg = %p (0x%llx dma), new deq ptr = %p (0x%llx dma), new cycle = %u",
4110		- deq_state->new_deq_seg,
4111		- (unsigned long long)deq_state->new_deq_seg->dma,
4112		- deq_state->new_deq_ptr,
4113		- (unsigned long long)xhci_trb_virt_to_dma(
4114		- deq_state->new_deq_seg, deq_state->new_deq_ptr),
4115		- deq_state->new_cycle_state);
4116		-
4117		- addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
4118		- deq_state->new_deq_ptr);
4119		- if (addr == 0) {
4120		- xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
4121		- xhci_warn(xhci, "WARN deq seg = %p, deq pt = %p\n",
4122		- deq_state->new_deq_seg, deq_state->new_deq_ptr);
4123		- return;
4124		- }
4125		- ep = &xhci->devs[slot_id]->eps[ep_index];
4126		- if ((ep->ep_state & SET_DEQ_PENDING)) {
4127		- xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
4128		- xhci_warn(xhci, "A Set TR Deq Ptr command is pending.\n");
4129		- return;
4130		- }
4131		-
4132		- /* This function gets called from contexts where it cannot sleep */
4133		- cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC);
4134		- if (!cmd)
4135		- return;
4136		-
4137		- ep->queued_deq_seg = deq_state->new_deq_seg;
4138		- ep->queued_deq_ptr = deq_state->new_deq_ptr;
4139		- if (deq_state->stream_id)
4140		- trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
4141		- ret = queue_command(xhci, cmd,
4142		- lower_32_bits(addr) \| trb_sct \| deq_state->new_cycle_state,
4143		- upper_32_bits(addr), trb_stream_id,
4144		- trb_slot_id \| trb_ep_index \| type, false);
4145		- if (ret < 0) {
4146		- xhci_free_command(xhci, cmd);
4147		- return;
4148		- }
4149		-
4150		- /* Stop the TD queueing code from ringing the doorbell until
4151		- * this command completes. The HC won't set the dequeue pointer
4152		- * if the ring is running, and ringing the doorbell starts the
4153		- * ring running.
4154		- */
4155		- ep->ep_state \|= SET_DEQ_PENDING;
4156		-}
	4458	+EXPORT_SYMBOL_GPL(xhci_queue_stop_endpoint);
4157	4459
4158	4460	int xhci_queue_reset_ep(struct xhci_hcd xhci, struct xhci_command cmd,
4159	4461	int slot_id, unsigned int ep_index,