~hc/RK356X_SDK_RELEASE.git

..	..	@@ -57,7 +57,10 @@
57	57	#include <linux/dma-mapping.h>
58	58	#include "xhci.h"
59	59	#include "xhci-trace.h"
60		-#include "xhci-mtk.h"
	60	+
	61	+static int queue_command(struct xhci_hcd xhci, struct xhci_command cmd,
	62	+ u32 field1, u32 field2,
	63	+ u32 field3, u32 field4, bool command_must_succeed);
61	64
62	65	/*
63	66	* Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
..	..	@@ -76,6 +79,7 @@
76	79	return 0;
77	80	return seg->dma + (segment_offset * sizeof(*trb));
78	81	}
	82	+EXPORT_SYMBOL_GPL(xhci_trb_virt_to_dma);
79	83
80	84	static bool trb_is_noop(union xhci_trb *trb)
81	85	{
..	..	@@ -151,10 +155,11 @@
151	155
152	156	/*
153	157	* See Cycle bit rules. SW is the consumer for the event ring only.
154		- * Don't make a ring full of link TRBs. That would be dumb and this would loop.
155	158	*/
156	159	void inc_deq(struct xhci_hcd xhci, struct xhci_ring ring)
157	160	{
	161	+ unsigned int link_trb_count = 0;
	162	+
158	163	/* event ring doesn't have link trbs, check for last trb */
159	164	if (ring->type == TYPE_EVENT) {
160	165	if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
..	..	@@ -170,14 +175,23 @@
170	175
171	176	/* All other rings have link trbs */
172	177	if (!trb_is_link(ring->dequeue)) {
173		- ring->dequeue++;
174		- ring->num_trbs_free++;
	178	+ if (last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
	179	+ xhci_warn(xhci, "Missing link TRB at end of segment\n");
	180	+ } else {
	181	+ ring->dequeue++;
	182	+ ring->num_trbs_free++;
	183	+ }
175	184	}
	185	+
176	186	while (trb_is_link(ring->dequeue)) {
177	187	ring->deq_seg = ring->deq_seg->next;
178	188	ring->dequeue = ring->deq_seg->trbs;
179		- }
180	189
	190	+ if (link_trb_count++ > ring->num_segs) {
	191	+ xhci_warn(xhci, "Ring is an endless link TRB loop\n");
	192	+ break;
	193	+ }
	194	+ }
181	195	out:
182	196	trace_xhci_inc_deq(ring);
183	197
..	..	@@ -186,7 +200,6 @@
186	200
187	201	/*
188	202	* See Cycle bit rules. SW is the consumer for the event ring only.
189		- * Don't make a ring full of link TRBs. That would be dumb and this would loop.
190	203	*
191	204	* If we've just enqueued a TRB that is in the middle of a TD (meaning the
192	205	* chain bit is set), then set the chain bit in all the following link TRBs.
..	..	@@ -206,11 +219,18 @@
206	219	{
207	220	u32 chain;
208	221	union xhci_trb *next;
	222	+ unsigned int link_trb_count = 0;
209	223
210	224	chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
211	225	/* If this is not event ring, there is one less usable TRB */
212	226	if (!trb_is_link(ring->enqueue))
213	227	ring->num_trbs_free--;
	228	+
	229	+ if (last_trb_on_seg(ring->enq_seg, ring->enqueue)) {
	230	+ xhci_err(xhci, "Tried to move enqueue past ring segment\n");
	231	+ return;
	232	+ }
	233	+
214	234	next = ++(ring->enqueue);
215	235
216	236	/* Update the dequeue pointer further if that was a link TRB */
..	..	@@ -247,6 +267,11 @@
247	267	ring->enq_seg = ring->enq_seg->next;
248	268	ring->enqueue = ring->enq_seg->trbs;
249	269	next = ring->enqueue;
	270	+
	271	+ if (link_trb_count++ > ring->num_segs) {
	272	+ xhci_warn(xhci, "%s: Ring link TRB loop\n", __func__);
	273	+ break;
	274	+ }
250	275	}
251	276
252	277	trace_xhci_inc_enq(ring);
..	..	@@ -280,10 +305,14 @@
280	305	return;
281	306
282	307	xhci_dbg(xhci, "// Ding dong!\n");
	308	+
	309	+ trace_xhci_ring_host_doorbell(0, DB_VALUE_HOST);
	310	+
283	311	writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]);
284	312	/* Flush PCI posted writes */
285	313	readl(&xhci->dba->doorbell[0]);
286	314	}
	315	+EXPORT_SYMBOL_GPL(xhci_ring_cmd_db);
287	316
288	317	static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay)
289	318	{
..	..	@@ -412,12 +441,14 @@
412	441	* stream once the endpoint is on the HW schedule.
413	442	*/
414	443	if ((ep_state & EP_STOP_CMD_PENDING) \|\| (ep_state & SET_DEQ_PENDING) \|\|
415		- (ep_state & EP_HALTED))
	444	+ (ep_state & EP_HALTED) \|\| (ep_state & EP_CLEARING_TT))
416	445	return;
	446	+
	447	+ trace_xhci_ring_ep_doorbell(slot_id, DB_VALUE(ep_index, stream_id));
	448	+
417	449	writel(DB_VALUE(ep_index, stream_id), db_addr);
418		- /* The CPU has better things to do at this point than wait for a
419		- * write-posting flush. It'll get there soon enough.
420		- */
	450	+ /* flush the write */
	451	+ readl(db_addr);
421	452	}
422	453
423	454	/* Ring the doorbell for any rings with pending URBs */
..	..	@@ -446,6 +477,13 @@
446	477	}
447	478	}
448	479
	480	+void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
	481	+ unsigned int slot_id,
	482	+ unsigned int ep_index)
	483	+{
	484	+ ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
	485	+}
	486	+
449	487	static struct xhci_virt_ep xhci_get_virt_ep(struct xhci_hcd xhci,
450	488	unsigned int slot_id,
451	489	unsigned int ep_index)
..	..	@@ -466,6 +504,26 @@
466	504	return &xhci->devs[slot_id]->eps[ep_index];
467	505	}
468	506
	507	+static struct xhci_ring xhci_virt_ep_to_ring(struct xhci_hcd xhci,
	508	+ struct xhci_virt_ep *ep,
	509	+ unsigned int stream_id)
	510	+{
	511	+ /* common case, no streams */
	512	+ if (!(ep->ep_state & EP_HAS_STREAMS))
	513	+ return ep->ring;
	514	+
	515	+ if (!ep->stream_info)
	516	+ return NULL;
	517	+
	518	+ if (stream_id == 0 \|\| stream_id >= ep->stream_info->num_streams) {
	519	+ xhci_warn(xhci, "Invalid stream_id %u request for slot_id %u ep_index %u\n",
	520	+ stream_id, ep->vdev->slot_id, ep->ep_index);
	521	+ return NULL;
	522	+ }
	523	+
	524	+ return ep->stream_info->stream_rings[stream_id];
	525	+}
	526	+
469	527	/* Get the right ring for the given slot_id, ep_index and stream_id.
470	528	* If the endpoint supports streams, boundary check the URB's stream ID.
471	529	* If the endpoint doesn't support streams, return the singular endpoint ring.
..	..	@@ -480,29 +538,7 @@
480	538	if (!ep)
481	539	return NULL;
482	540
483		- /* Common case: no streams */
484		- if (!(ep->ep_state & EP_HAS_STREAMS))
485		- return ep->ring;
486		-
487		- if (stream_id == 0) {
488		- xhci_warn(xhci,
489		- "WARN: Slot ID %u, ep index %u has streams, "
490		- "but URB has no stream ID.\n",
491		- slot_id, ep_index);
492		- return NULL;
493		- }
494		-
495		- if (stream_id < ep->stream_info->num_streams)
496		- return ep->stream_info->stream_rings[stream_id];
497		-
498		- xhci_warn(xhci,
499		- "WARN: Slot ID %u, ep index %u has "
500		- "stream IDs 1 to %u allocated, "
501		- "but stream ID %u is requested.\n",
502		- slot_id, ep_index,
503		- ep->stream_info->num_streams - 1,
504		- stream_id);
505		- return NULL;
	541	+ return xhci_virt_ep_to_ring(xhci, ep, stream_id);
506	542	}
507	543
508	544
..	..	@@ -529,56 +565,78 @@
529	565	return le64_to_cpu(ep_ctx->deq);
530	566	}
531	567
532		-/*
533		- * Move the xHC's endpoint ring dequeue pointer past cur_td.
534		- * Record the new state of the xHC's endpoint ring dequeue segment,
535		- * dequeue pointer, stream id, and new consumer cycle state in state.
536		- * Update our internal representation of the ring's dequeue pointer.
537		- *
538		- * We do this in three jumps:
539		- * - First we update our new ring state to be the same as when the xHC stopped.
540		- * - Then we traverse the ring to find the segment that contains
541		- * the last TRB in the TD. We toggle the xHC's new cycle state when we pass
542		- * any link TRBs with the toggle cycle bit set.
543		- * - Finally we move the dequeue state one TRB further, toggling the cycle bit
544		- * if we've moved it past a link TRB with the toggle cycle bit set.
545		- *
546		- * Some of the uses of xhci_generic_trb are grotty, but if they're done
547		- * with correct __le32 accesses they should work fine. Only users of this are
548		- * in here.
549		- */
550		-void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
551		- unsigned int slot_id, unsigned int ep_index,
552		- unsigned int stream_id, struct xhci_td *cur_td,
553		- struct xhci_dequeue_state *state)
	568	+static int xhci_move_dequeue_past_td(struct xhci_hcd *xhci,
	569	+ unsigned int slot_id, unsigned int ep_index,
	570	+ unsigned int stream_id, struct xhci_td *td)
554	571	{
555	572	struct xhci_virt_device *dev = xhci->devs[slot_id];
556	573	struct xhci_virt_ep *ep = &dev->eps[ep_index];
557	574	struct xhci_ring *ep_ring;
	575	+ struct xhci_command *cmd;
558	576	struct xhci_segment *new_seg;
	577	+ struct xhci_segment *halted_seg = NULL;
559	578	union xhci_trb *new_deq;
	579	+ int new_cycle;
	580	+ union xhci_trb *halted_trb;
	581	+ int index = 0;
560	582	dma_addr_t addr;
561	583	u64 hw_dequeue;
562	584	bool cycle_found = false;
563	585	bool td_last_trb_found = false;
	586	+ u32 trb_sct = 0;
	587	+ int ret;
564	588
565	589	ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,
566	590	ep_index, stream_id);
567	591	if (!ep_ring) {
568		- xhci_warn(xhci, "WARN can't find new dequeue state "
569		- "for invalid stream ID %u.\n",
570		- stream_id);
571		- return;
	592	+ xhci_warn(xhci, "WARN can't find new dequeue, invalid stream ID %u\n",
	593	+ stream_id);
	594	+ return -ENODEV;
572	595	}
573		- /* Dig out the cycle state saved by the xHC during the stop ep cmd */
574		- xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
575		- "Finding endpoint context");
	596	+ /*
	597	+ * A cancelled TD can complete with a stall if HW cached the trb.
	598	+ * In this case driver can't find td, but if the ring is empty we
	599	+ * can move the dequeue pointer to the current enqueue position.
	600	+ * We shouldn't hit this anymore as cached cancelled TRBs are given back
	601	+ * after clearing the cache, but be on the safe side and keep it anyway
	602	+ */
	603	+ if (!td) {
	604	+ if (list_empty(&ep_ring->td_list)) {
	605	+ new_seg = ep_ring->enq_seg;
	606	+ new_deq = ep_ring->enqueue;
	607	+ new_cycle = ep_ring->cycle_state;
	608	+ xhci_dbg(xhci, "ep ring empty, Set new dequeue = enqueue");
	609	+ goto deq_found;
	610	+ } else {
	611	+ xhci_warn(xhci, "Can't find new dequeue state, missing td\n");
	612	+ return -EINVAL;
	613	+ }
	614	+ }
576	615
577	616	hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id);
578	617	new_seg = ep_ring->deq_seg;
579	618	new_deq = ep_ring->dequeue;
580		- state->new_cycle_state = hw_dequeue & 0x1;
581		- state->stream_id = stream_id;
	619	+
	620	+ /*
	621	+ * Quirk: xHC write-back of the DCS field in the hardware dequeue
	622	+ * pointer is wrong - use the cycle state of the TRB pointed to by
	623	+ * the dequeue pointer.
	624	+ */
	625	+ if (xhci->quirks & XHCI_EP_CTX_BROKEN_DCS &&
	626	+ !(ep->ep_state & EP_HAS_STREAMS))
	627	+ halted_seg = trb_in_td(xhci, td->start_seg,
	628	+ td->first_trb, td->last_trb,
	629	+ hw_dequeue & ~0xf, false);
	630	+ if (halted_seg) {
	631	+ index = ((dma_addr_t)(hw_dequeue & ~0xf) - halted_seg->dma) /
	632	+ sizeof(*halted_trb);
	633	+ halted_trb = &halted_seg->trbs[index];
	634	+ new_cycle = halted_trb->generic.field[3] & 0x1;
	635	+ xhci_dbg(xhci, "Endpoint DCS = %d TRB index = %d cycle = %d\n",
	636	+ (u8)(hw_dequeue & 0x1), index, new_cycle);
	637	+ } else {
	638	+ new_cycle = hw_dequeue & 0x1;
	639	+ }
582	640
583	641	/*
584	642	* We want to find the pointer, segment and cycle state of the new trb
..	..	@@ -593,39 +651,71 @@
593	651	if (td_last_trb_found)
594	652	break;
595	653	}
596		- if (new_deq == cur_td->last_trb)
	654	+ if (new_deq == td->last_trb)
597	655	td_last_trb_found = true;
598	656
599	657	if (cycle_found && trb_is_link(new_deq) &&
600	658	link_trb_toggles_cycle(new_deq))
601		- state->new_cycle_state ^= 0x1;
	659	+ new_cycle ^= 0x1;
602	660
603	661	next_trb(xhci, ep_ring, &new_seg, &new_deq);
604	662
605	663	/* Search wrapped around, bail out */
606	664	if (new_deq == ep->ring->dequeue) {
607	665	xhci_err(xhci, "Error: Failed finding new dequeue state\n");
608		- state->new_deq_seg = NULL;
609		- state->new_deq_ptr = NULL;
610		- return;
	666	+ return -EINVAL;
611	667	}
612	668
613	669	} while (!cycle_found \|\| !td_last_trb_found);
614	670
615		- state->new_deq_seg = new_seg;
616		- state->new_deq_ptr = new_deq;
	671	+deq_found:
617	672
618	673	/* Don't update the ring cycle state for the producer (us). */
619		- xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
620		- "Cycle state = 0x%x", state->new_cycle_state);
	674	+ addr = xhci_trb_virt_to_dma(new_seg, new_deq);
	675	+ if (addr == 0) {
	676	+ xhci_warn(xhci, "Can't find dma of new dequeue ptr\n");
	677	+ xhci_warn(xhci, "deq seg = %p, deq ptr = %p\n", new_seg, new_deq);
	678	+ return -EINVAL;
	679	+ }
	680	+
	681	+ if ((ep->ep_state & SET_DEQ_PENDING)) {
	682	+ xhci_warn(xhci, "Set TR Deq already pending, don't submit for 0x%pad\n",
	683	+ &addr);
	684	+ return -EBUSY;
	685	+ }
	686	+
	687	+ /* This function gets called from contexts where it cannot sleep */
	688	+ cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC);
	689	+ if (!cmd) {
	690	+ xhci_warn(xhci, "Can't alloc Set TR Deq cmd 0x%pad\n", &addr);
	691	+ return -ENOMEM;
	692	+ }
	693	+
	694	+ if (stream_id)
	695	+ trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
	696	+ ret = queue_command(xhci, cmd,
	697	+ lower_32_bits(addr) \| trb_sct \| new_cycle,
	698	+ upper_32_bits(addr),
	699	+ STREAM_ID_FOR_TRB(stream_id), SLOT_ID_FOR_TRB(slot_id) \|
	700	+ EP_ID_FOR_TRB(ep_index) \| TRB_TYPE(TRB_SET_DEQ), false);
	701	+ if (ret < 0) {
	702	+ xhci_free_command(xhci, cmd);
	703	+ return ret;
	704	+ }
	705	+ ep->queued_deq_seg = new_seg;
	706	+ ep->queued_deq_ptr = new_deq;
621	707
622	708	xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
623		- "New dequeue segment = %p (virtual)",
624		- state->new_deq_seg);
625		- addr = xhci_trb_virt_to_dma(state->new_deq_seg, state->new_deq_ptr);
626		- xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
627		- "New dequeue pointer = 0x%llx (DMA)",
628		- (unsigned long long) addr);
	709	+ "Set TR Deq ptr 0x%llx, cycle %u\n", addr, new_cycle);
	710	+
	711	+ /* Stop the TD queueing code from ringing the doorbell until
	712	+ * this command completes. The HC won't set the dequeue pointer
	713	+ * if the ring is running, and ringing the doorbell starts the
	714	+ * ring running.
	715	+ */
	716	+ ep->ep_state \|= SET_DEQ_PENDING;
	717	+ xhci_ring_cmd_db(xhci);
	718	+ return 0;
629	719	}
630	720
631	721	/* flip_cycle means flip the cycle bit of all but the first and last TRB.
..	..	@@ -680,10 +770,8 @@
680	770	}
681	771	xhci_urb_free_priv(urb_priv);
682	772	usb_hcd_unlink_urb_from_ep(hcd, urb);
683		- spin_unlock(&xhci->lock);
684	773	trace_xhci_urb_giveback(urb);
685	774	usb_hcd_giveback_urb(hcd, urb, status);
686		- spin_lock(&xhci->lock);
687	775	}
688	776
689	777	static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
..	..	@@ -720,6 +808,256 @@
720	808	seg->bounce_offs = 0;
721	809	}
722	810
	811	+static int xhci_td_cleanup(struct xhci_hcd xhci, struct xhci_td td,
	812	+ struct xhci_ring *ep_ring, int status)
	813	+{
	814	+ struct urb *urb = NULL;
	815	+
	816	+ /* Clean up the endpoint's TD list */
	817	+ urb = td->urb;
	818	+
	819	+ /* if a bounce buffer was used to align this td then unmap it */
	820	+ xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);
	821	+
	822	+ /* Do one last check of the actual transfer length.
	823	+ * If the host controller said we transferred more data than the buffer
	824	+ * length, urb->actual_length will be a very big number (since it's
	825	+ * unsigned). Play it safe and say we didn't transfer anything.
	826	+ */
	827	+ if (urb->actual_length > urb->transfer_buffer_length) {
	828	+ xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
	829	+ urb->transfer_buffer_length, urb->actual_length);
	830	+ urb->actual_length = 0;
	831	+ status = 0;
	832	+ }
	833	+ /* TD might be removed from td_list if we are giving back a cancelled URB */
	834	+ if (!list_empty(&td->td_list))
	835	+ list_del_init(&td->td_list);
	836	+ /* Giving back a cancelled URB, or if a slated TD completed anyway */
	837	+ if (!list_empty(&td->cancelled_td_list))
	838	+ list_del_init(&td->cancelled_td_list);
	839	+
	840	+ inc_td_cnt(urb);
	841	+ /* Giveback the urb when all the tds are completed */
	842	+ if (last_td_in_urb(td)) {
	843	+ if ((urb->actual_length != urb->transfer_buffer_length &&
	844	+ (urb->transfer_flags & URB_SHORT_NOT_OK)) \|\|
	845	+ (status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc)))
	846	+ xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
	847	+ urb, urb->actual_length,
	848	+ urb->transfer_buffer_length, status);
	849	+
	850	+ /* set isoc urb status to 0 just as EHCI, UHCI, and OHCI */
	851	+ if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
	852	+ status = 0;
	853	+ xhci_giveback_urb_in_irq(xhci, td, status);
	854	+ }
	855	+
	856	+ return 0;
	857	+}
	858	+
	859	+
	860	+/* Complete the cancelled URBs we unlinked from td_list. */
	861	+static void xhci_giveback_invalidated_tds(struct xhci_virt_ep *ep)
	862	+{
	863	+ struct xhci_ring *ring;
	864	+ struct xhci_td td, tmp_td;
	865	+
	866	+ list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list,
	867	+ cancelled_td_list) {
	868	+
	869	+ ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb);
	870	+
	871	+ if (td->cancel_status == TD_CLEARED) {
	872	+ xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n",
	873	+ __func__, td->urb);
	874	+ xhci_td_cleanup(ep->xhci, td, ring, td->status);
	875	+ } else {
	876	+ xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n",
	877	+ __func__, td->urb, td->cancel_status);
	878	+ }
	879	+ if (ep->xhci->xhc_state & XHCI_STATE_DYING)
	880	+ return;
	881	+ }
	882	+}
	883	+
	884	+static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id,
	885	+ unsigned int ep_index, enum xhci_ep_reset_type reset_type)
	886	+{
	887	+ struct xhci_command *command;
	888	+ int ret = 0;
	889	+
	890	+ command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
	891	+ if (!command) {
	892	+ ret = -ENOMEM;
	893	+ goto done;
	894	+ }
	895	+
	896	+ xhci_dbg(xhci, "%s-reset ep %u, slot %u\n",
	897	+ (reset_type == EP_HARD_RESET) ? "Hard" : "Soft",
	898	+ ep_index, slot_id);
	899	+
	900	+ ret = xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type);
	901	+done:
	902	+ if (ret)
	903	+ xhci_err(xhci, "ERROR queuing reset endpoint for slot %d ep_index %d, %d\n",
	904	+ slot_id, ep_index, ret);
	905	+ return ret;
	906	+}
	907	+
	908	+static int xhci_handle_halted_endpoint(struct xhci_hcd *xhci,
	909	+ struct xhci_virt_ep *ep, unsigned int stream_id,
	910	+ struct xhci_td *td,
	911	+ enum xhci_ep_reset_type reset_type)
	912	+{
	913	+ unsigned int slot_id = ep->vdev->slot_id;
	914	+ int err;
	915	+
	916	+ /*
	917	+ * Avoid resetting endpoint if link is inactive. Can cause host hang.
	918	+ * Device will be reset soon to recover the link so don't do anything
	919	+ */
	920	+ if (ep->vdev->flags & VDEV_PORT_ERROR)
	921	+ return -ENODEV;
	922	+
	923	+ /* add td to cancelled list and let reset ep handler take care of it */
	924	+ if (reset_type == EP_HARD_RESET) {
	925	+ ep->ep_state \|= EP_HARD_CLEAR_TOGGLE;
	926	+ if (td && list_empty(&td->cancelled_td_list)) {
	927	+ list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list);
	928	+ td->cancel_status = TD_HALTED;
	929	+ }
	930	+ }
	931	+
	932	+ if (ep->ep_state & EP_HALTED) {
	933	+ xhci_dbg(xhci, "Reset ep command for ep_index %d already pending\n",
	934	+ ep->ep_index);
	935	+ return 0;
	936	+ }
	937	+
	938	+ err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type);
	939	+ if (err)
	940	+ return err;
	941	+
	942	+ ep->ep_state \|= EP_HALTED;
	943	+
	944	+ xhci_ring_cmd_db(xhci);
	945	+
	946	+ return 0;
	947	+}
	948	+
	949	+/*
	950	+ * Fix up the ep ring first, so HW stops executing cancelled TDs.
	951	+ * We have the xHCI lock, so nothing can modify this list until we drop it.
	952	+ * We're also in the event handler, so we can't get re-interrupted if another
	953	+ * Stop Endpoint command completes.
	954	+ *
	955	+ * only call this when ring is not in a running state
	956	+ */
	957	+
	958	+static int xhci_invalidate_cancelled_tds(struct xhci_virt_ep *ep)
	959	+{
	960	+ struct xhci_hcd *xhci;
	961	+ struct xhci_td *td = NULL;
	962	+ struct xhci_td *tmp_td = NULL;
	963	+ struct xhci_td *cached_td = NULL;
	964	+ struct xhci_ring *ring;
	965	+ u64 hw_deq;
	966	+ unsigned int slot_id = ep->vdev->slot_id;
	967	+ int err;
	968	+
	969	+ xhci = ep->xhci;
	970	+
	971	+ list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) {
	972	+ xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
	973	+ "Removing canceled TD starting at 0x%llx (dma) in stream %u URB %p",
	974	+ (unsigned long long)xhci_trb_virt_to_dma(
	975	+ td->start_seg, td->first_trb),
	976	+ td->urb->stream_id, td->urb);
	977	+ list_del_init(&td->td_list);
	978	+ ring = xhci_urb_to_transfer_ring(xhci, td->urb);
	979	+ if (!ring) {
	980	+ xhci_warn(xhci, "WARN Cancelled URB %p has invalid stream ID %u.\n",
	981	+ td->urb, td->urb->stream_id);
	982	+ continue;
	983	+ }
	984	+ /*
	985	+ * If a ring stopped on the TD we need to cancel then we have to
	986	+ * move the xHC endpoint ring dequeue pointer past this TD.
	987	+ * Rings halted due to STALL may show hw_deq is past the stalled
	988	+ * TD, but still require a set TR Deq command to flush xHC cache.
	989	+ */
	990	+ hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index,
	991	+ td->urb->stream_id);
	992	+ hw_deq &= ~0xf;
	993	+
	994	+ if (td->cancel_status == TD_HALTED \|\|
	995	+ trb_in_td(xhci, td->start_seg, td->first_trb, td->last_trb, hw_deq, false)) {
	996	+ switch (td->cancel_status) {
	997	+ case TD_CLEARED: /* TD is already no-op */
	998	+ case TD_CLEARING_CACHE: /* set TR deq command already queued */
	999	+ break;
	1000	+ case TD_DIRTY: /* TD is cached, clear it */
	1001	+ case TD_HALTED:
	1002	+ td->cancel_status = TD_CLEARING_CACHE;
	1003	+ if (cached_td)
	1004	+ /* FIXME stream case, several stopped rings */
	1005	+ xhci_dbg(xhci,
	1006	+ "Move dq past stream %u URB %p instead of stream %u URB %p\n",
	1007	+ td->urb->stream_id, td->urb,
	1008	+ cached_td->urb->stream_id, cached_td->urb);
	1009	+ cached_td = td;
	1010	+ ring->num_trbs_free += td->num_trbs;
	1011	+ break;
	1012	+ }
	1013	+ } else {
	1014	+ td_to_noop(xhci, ring, td, false);
	1015	+ td->cancel_status = TD_CLEARED;
	1016	+ ring->num_trbs_free += td->num_trbs;
	1017	+ }
	1018	+ }
	1019	+
	1020	+ /* If there's no need to move the dequeue pointer then we're done */
	1021	+ if (!cached_td)
	1022	+ return 0;
	1023	+
	1024	+ err = xhci_move_dequeue_past_td(xhci, slot_id, ep->ep_index,
	1025	+ cached_td->urb->stream_id,
	1026	+ cached_td);
	1027	+ if (err) {
	1028	+ /* Failed to move past cached td, just set cached TDs to no-op */
	1029	+ list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) {
	1030	+ if (td->cancel_status != TD_CLEARING_CACHE)
	1031	+ continue;
	1032	+ xhci_dbg(xhci, "Failed to clear cancelled cached URB %p, mark clear anyway\n",
	1033	+ td->urb);
	1034	+ td_to_noop(xhci, ring, td, false);
	1035	+ td->cancel_status = TD_CLEARED;
	1036	+ }
	1037	+ }
	1038	+ return 0;
	1039	+}
	1040	+
	1041	+/*
	1042	+ * Returns the TD the endpoint ring halted on.
	1043	+ * Only call for non-running rings without streams.
	1044	+ */
	1045	+static struct xhci_td find_halted_td(struct xhci_virt_ep ep)
	1046	+{
	1047	+ struct xhci_td *td;
	1048	+ u64 hw_deq;
	1049	+
	1050	+ if (!list_empty(&ep->ring->td_list)) { /* Not streams compatible */
	1051	+ hw_deq = xhci_get_hw_deq(ep->xhci, ep->vdev, ep->ep_index, 0);
	1052	+ hw_deq &= ~0xf;
	1053	+ td = list_first_entry(&ep->ring->td_list, struct xhci_td, td_list);
	1054	+ if (trb_in_td(ep->xhci, td->start_seg, td->first_trb,
	1055	+ td->last_trb, hw_deq, false))
	1056	+ return td;
	1057	+ }
	1058	+ return NULL;
	1059	+}
	1060	+
723	1061	/*
724	1062	* When we get a command completion for a Stop Endpoint Command, we need to
725	1063	* unlink any cancelled TDs from the ring. There are two ways to do that:
..	..	@@ -731,142 +1069,90 @@
731	1069	* bit cleared) so that the HW will skip over them.
732	1070	*/
733	1071	static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
734		- union xhci_trb trb, struct xhci_event_cmd event)
	1072	+ union xhci_trb *trb, u32 comp_code)
735	1073	{
736	1074	unsigned int ep_index;
737		- struct xhci_ring *ep_ring;
738	1075	struct xhci_virt_ep *ep;
739		- struct xhci_td *cur_td = NULL;
740		- struct xhci_td *last_unlinked_td;
741	1076	struct xhci_ep_ctx *ep_ctx;
742		- struct xhci_virt_device *vdev;
743		- u64 hw_deq;
744		- struct xhci_dequeue_state deq_state;
	1077	+ struct xhci_td *td = NULL;
	1078	+ enum xhci_ep_reset_type reset_type;
	1079	+ struct xhci_command *command;
	1080	+ int err;
745	1081
746	1082	if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) {
747	1083	if (!xhci->devs[slot_id])
748		- xhci_warn(xhci, "Stop endpoint command "
749		- "completion for disabled slot %u\n",
750		- slot_id);
	1084	+ xhci_warn(xhci, "Stop endpoint command completion for disabled slot %u\n",
	1085	+ slot_id);
751	1086	return;
752	1087	}
753	1088
754		- memset(&deq_state, 0, sizeof(deq_state));
755	1089	ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
756		-
757	1090	ep = xhci_get_virt_ep(xhci, slot_id, ep_index);
758	1091	if (!ep)
759	1092	return;
760	1093
761		- vdev = xhci->devs[slot_id];
762		- ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
	1094	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
	1095	+
763	1096	trace_xhci_handle_cmd_stop_ep(ep_ctx);
764	1097
765		- last_unlinked_td = list_last_entry(&ep->cancelled_td_list,
766		- struct xhci_td, cancelled_td_list);
767		-
768		- if (list_empty(&ep->cancelled_td_list)) {
769		- xhci_stop_watchdog_timer_in_irq(xhci, ep);
770		- ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
771		- return;
772		- }
773		-
774		- /* Fix up the ep ring first, so HW stops executing cancelled TDs.
775		- * We have the xHCI lock, so nothing can modify this list until we drop
776		- * it. We're also in the event handler, so we can't get re-interrupted
777		- * if another Stop Endpoint command completes
	1098	+ if (comp_code == COMP_CONTEXT_STATE_ERROR) {
	1099	+ /*
	1100	+ * If stop endpoint command raced with a halting endpoint we need to
	1101	+ * reset the host side endpoint first.
	1102	+ * If the TD we halted on isn't cancelled the TD should be given back
	1103	+ * with a proper error code, and the ring dequeue moved past the TD.
	1104	+ * If streams case we can't find hw_deq, or the TD we halted on so do a
	1105	+ * soft reset.
	1106	+ *
	1107	+ * Proper error code is unknown here, it would be -EPIPE if device side
	1108	+ * of enadpoit halted (aka STALL), and -EPROTO if not (transaction error)
	1109	+ * We use -EPROTO, if device is stalled it should return a stall error on
	1110	+ * next transfer, which then will return -EPIPE, and device side stall is
	1111	+ * noted and cleared by class driver.
778	1112	*/
779		- list_for_each_entry(cur_td, &ep->cancelled_td_list, cancelled_td_list) {
780		- xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
781		- "Removing canceled TD starting at 0x%llx (dma).",
782		- (unsigned long long)xhci_trb_virt_to_dma(
783		- cur_td->start_seg, cur_td->first_trb));
784		- ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
785		- if (!ep_ring) {
786		- /* This shouldn't happen unless a driver is mucking
787		- * with the stream ID after submission. This will
788		- * leave the TD on the hardware ring, and the hardware
789		- * will try to execute it, and may access a buffer
790		- * that has already been freed. In the best case, the
791		- * hardware will execute it, and the event handler will
792		- * ignore the completion event for that TD, since it was
793		- * removed from the td_list for that endpoint. In
794		- * short, don't muck with the stream ID after
795		- * submission.
796		- */
797		- xhci_warn(xhci, "WARN Cancelled URB %p "
798		- "has invalid stream ID %u.\n",
799		- cur_td->urb,
800		- cur_td->urb->stream_id);
801		- goto remove_finished_td;
802		- }
803		- /*
804		- * If we stopped on the TD we need to cancel, then we have to
805		- * move the xHC endpoint ring dequeue pointer past this TD.
806		- */
807		- hw_deq = xhci_get_hw_deq(xhci, vdev, ep_index,
808		- cur_td->urb->stream_id);
809		- hw_deq &= ~0xf;
	1113	+ switch (GET_EP_CTX_STATE(ep_ctx)) {
	1114	+ case EP_STATE_HALTED:
	1115	+ xhci_dbg(xhci, "Stop ep completion raced with stall, reset ep\n");
	1116	+ if (ep->ep_state & EP_HAS_STREAMS) {
	1117	+ reset_type = EP_SOFT_RESET;
	1118	+ } else {
	1119	+ reset_type = EP_HARD_RESET;
	1120	+ td = find_halted_td(ep);
	1121	+ if (td)
	1122	+ td->status = -EPROTO;
	1123	+ }
	1124	+ /* reset ep, reset handler cleans up cancelled tds */
	1125	+ err = xhci_handle_halted_endpoint(xhci, ep, 0, td,
	1126	+ reset_type);
	1127	+ if (err)
	1128	+ break;
	1129	+ xhci_stop_watchdog_timer_in_irq(xhci, ep);
	1130	+ return;
	1131	+ case EP_STATE_RUNNING:
	1132	+ /* Race, HW handled stop ep cmd before ep was running */
	1133	+ xhci_dbg(xhci, "Stop ep completion ctx error, ep is running\n");
810	1134
811		- if (trb_in_td(xhci, cur_td->start_seg, cur_td->first_trb,
812		- cur_td->last_trb, hw_deq, false)) {
813		- xhci_find_new_dequeue_state(xhci, slot_id, ep_index,
814		- cur_td->urb->stream_id,
815		- cur_td, &deq_state);
816		- } else {
817		- td_to_noop(xhci, ep_ring, cur_td, false);
818		- }
	1135	+ command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
	1136	+ if (!command)
	1137	+ xhci_stop_watchdog_timer_in_irq(xhci, ep);
819	1138
820		-remove_finished_td:
821		- /*
822		- * The event handler won't see a completion for this TD anymore,
823		- * so remove it from the endpoint ring's TD list. Keep it in
824		- * the cancelled TD list for URB completion later.
825		- */
826		- list_del_init(&cur_td->td_list);
	1139	+ mod_timer(&ep->stop_cmd_timer,
	1140	+ jiffies + XHCI_STOP_EP_CMD_TIMEOUT * HZ);
	1141	+ xhci_queue_stop_endpoint(xhci, command, slot_id, ep_index, 0);
	1142	+ xhci_ring_cmd_db(xhci);
	1143	+
	1144	+ return;
	1145	+ default:
	1146	+ break;
	1147	+ }
827	1148	}
828		-
	1149	+ /* will queue a set TR deq if stopped on a cancelled, uncleared TD */
	1150	+ xhci_invalidate_cancelled_tds(ep);
829	1151	xhci_stop_watchdog_timer_in_irq(xhci, ep);
830	1152
831		- /* If necessary, queue a Set Transfer Ring Dequeue Pointer command */
832		- if (deq_state.new_deq_ptr && deq_state.new_deq_seg) {
833		- xhci_queue_new_dequeue_state(xhci, slot_id, ep_index,
834		- &deq_state);
835		- xhci_ring_cmd_db(xhci);
836		- } else {
837		- /* Otherwise ring the doorbell(s) to restart queued transfers */
838		- ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
839		- }
840		-
841		- /*
842		- * Drop the lock and complete the URBs in the cancelled TD list.
843		- * New TDs to be cancelled might be added to the end of the list before
844		- * we can complete all the URBs for the TDs we already unlinked.
845		- * So stop when we've completed the URB for the last TD we unlinked.
846		- */
847		- do {
848		- cur_td = list_first_entry(&ep->cancelled_td_list,
849		- struct xhci_td, cancelled_td_list);
850		- list_del_init(&cur_td->cancelled_td_list);
851		-
852		- /* Clean up the cancelled URB */
853		- /* Doesn't matter what we pass for status, since the core will
854		- * just overwrite it (because the URB has been unlinked).
855		- */
856		- ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
857		- xhci_unmap_td_bounce_buffer(xhci, ep_ring, cur_td);
858		- inc_td_cnt(cur_td->urb);
859		- if (last_td_in_urb(cur_td))
860		- xhci_giveback_urb_in_irq(xhci, cur_td, 0);
861		-
862		- /* Stop processing the cancelled list if the watchdog timer is
863		- * running.
864		- */
865		- if (xhci->xhc_state & XHCI_STATE_DYING)
866		- return;
867		- } while (cur_td != last_unlinked_td);
868		-
869		- /* Return to the event handler with xhci->lock re-acquired */
	1153	+ /* Otherwise ring the doorbell(s) to restart queued transfers */
	1154	+ xhci_giveback_invalidated_tds(ep);
	1155	+ ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
870	1156	}
871	1157
872	1158	static void xhci_kill_ring_urbs(struct xhci_hcd xhci, struct xhci_ring ring)
..	..	@@ -988,6 +1274,8 @@
988	1274	struct xhci_virt_ep *ep = from_timer(ep, t, stop_cmd_timer);
989	1275	struct xhci_hcd *xhci = ep->xhci;
990	1276	unsigned long flags;
	1277	+ u32 usbsts;
	1278	+ char str[XHCI_MSG_MAX];
991	1279
992	1280	spin_lock_irqsave(&xhci->lock, flags);
993	1281
..	..	@@ -998,8 +1286,11 @@
998	1286	xhci_dbg(xhci, "Stop EP timer raced with cmd completion, exit");
999	1287	return;
1000	1288	}
	1289	+ usbsts = readl(&xhci->op_regs->status);
1001	1290
1002	1291	xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n");
	1292	+ xhci_warn(xhci, "USBSTS:%s\n", xhci_decode_usbsts(str, usbsts));
	1293	+
1003	1294	ep->ep_state &= ~EP_STOP_CMD_PENDING;
1004	1295
1005	1296	xhci_halt(xhci);
..	..	@@ -1022,10 +1313,7 @@
1022	1313	unsigned int ep_index)
1023	1314	{
1024	1315	union xhci_trb *dequeue_temp;
1025		- int num_trbs_free_temp;
1026		- bool revert = false;
1027	1316
1028		- num_trbs_free_temp = ep_ring->num_trbs_free;
1029	1317	dequeue_temp = ep_ring->dequeue;
1030	1318
1031	1319	/* If we get two back-to-back stalls, and the first stalled transfer
..	..	@@ -1040,8 +1328,6 @@
1040	1328	}
1041	1329
1042	1330	while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) {
1043		- /* We have more usable TRBs */
1044		- ep_ring->num_trbs_free++;
1045	1331	ep_ring->dequeue++;
1046	1332	if (trb_is_link(ep_ring->dequeue)) {
1047	1333	if (ep_ring->dequeue ==
..	..	@@ -1051,14 +1337,9 @@
1051	1337	ep_ring->dequeue = ep_ring->deq_seg->trbs;
1052	1338	}
1053	1339	if (ep_ring->dequeue == dequeue_temp) {
1054		- revert = true;
	1340	+ xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
1055	1341	break;
1056	1342	}
1057		- }
1058		-
1059		- if (revert) {
1060		- xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
1061		- ep_ring->num_trbs_free = num_trbs_free_temp;
1062	1343	}
1063	1344	}
1064	1345
..	..	@@ -1075,10 +1356,10 @@
1075	1356	unsigned int ep_index;
1076	1357	unsigned int stream_id;
1077	1358	struct xhci_ring *ep_ring;
1078		- struct xhci_virt_device *dev;
1079	1359	struct xhci_virt_ep *ep;
1080	1360	struct xhci_ep_ctx *ep_ctx;
1081	1361	struct xhci_slot_ctx *slot_ctx;
	1362	+ struct xhci_td td, tmp_td;
1082	1363
1083	1364	ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1084	1365	stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2]));
..	..	@@ -1086,8 +1367,7 @@
1086	1367	if (!ep)
1087	1368	return;
1088	1369
1089		- dev = xhci->devs[slot_id];
1090		- ep_ring = xhci_stream_id_to_ring(dev, ep_index, stream_id);
	1370	+ ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id);
1091	1371	if (!ep_ring) {
1092	1372	xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n",
1093	1373	stream_id);
..	..	@@ -1095,8 +1375,8 @@
1095	1375	goto cleanup;
1096	1376	}
1097	1377
1098		- ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
1099		- slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx);
	1378	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
	1379	+ slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx);
1100	1380	trace_xhci_handle_cmd_set_deq(slot_ctx);
1101	1381	trace_xhci_handle_cmd_set_deq_ep(ep_ctx);
1102	1382
..	..	@@ -1149,7 +1429,7 @@
1149	1429	/* Update the ring's dequeue segment and dequeue pointer
1150	1430	* to reflect the new position.
1151	1431	*/
1152		- update_ring_for_set_deq_completion(xhci, dev,
	1432	+ update_ring_for_set_deq_completion(xhci, ep->vdev,
1153	1433	ep_ring, ep_index);
1154	1434	} else {
1155	1435	xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n");
..	..	@@ -1157,7 +1437,20 @@
1157	1437	ep->queued_deq_seg, ep->queued_deq_ptr);
1158	1438	}
1159	1439	}
1160		-
	1440	+ /* HW cached TDs cleared from cache, give them back */
	1441	+ list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list,
	1442	+ cancelled_td_list) {
	1443	+ ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb);
	1444	+ if (td->cancel_status == TD_CLEARING_CACHE) {
	1445	+ td->cancel_status = TD_CLEARED;
	1446	+ xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n",
	1447	+ __func__, td->urb);
	1448	+ xhci_td_cleanup(ep->xhci, td, ep_ring, td->status);
	1449	+ } else {
	1450	+ xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n",
	1451	+ __func__, td->urb, td->cancel_status);
	1452	+ }
	1453	+ }
1161	1454	cleanup:
1162	1455	ep->ep_state &= ~SET_DEQ_PENDING;
1163	1456	ep->queued_deq_seg = NULL;
..	..	@@ -1169,7 +1462,6 @@
1169	1462	static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
1170	1463	union xhci_trb *trb, u32 cmd_comp_code)
1171	1464	{
1172		- struct xhci_virt_device *vdev;
1173	1465	struct xhci_virt_ep *ep;
1174	1466	struct xhci_ep_ctx *ep_ctx;
1175	1467	unsigned int ep_index;
..	..	@@ -1179,8 +1471,7 @@
1179	1471	if (!ep)
1180	1472	return;
1181	1473
1182		- vdev = xhci->devs[slot_id];
1183		- ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
	1474	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
1184	1475	trace_xhci_handle_cmd_reset_ep(ep_ctx);
1185	1476
1186	1477	/* This command will only fail if the endpoint wasn't halted,
..	..	@@ -1189,27 +1480,19 @@
1189	1480	xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
1190	1481	"Ignoring reset ep completion code of %u", cmd_comp_code);
1191	1482
1192		- /* HW with the reset endpoint quirk needs to have a configure endpoint
1193		- * command complete before the endpoint can be used. Queue that here
1194		- * because the HW can't handle two commands being queued in a row.
1195		- */
1196		- if (xhci->quirks & XHCI_RESET_EP_QUIRK) {
1197		- struct xhci_command *command;
	1483	+ /* Cleanup cancelled TDs as ep is stopped. May queue a Set TR Deq cmd */
	1484	+ xhci_invalidate_cancelled_tds(ep);
1198	1485
1199		- command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1200		- if (!command)
1201		- return;
	1486	+ if (xhci->quirks & XHCI_RESET_EP_QUIRK)
	1487	+ xhci_dbg(xhci, "Note: Removed workaround to queue config ep for this hw");
	1488	+ /* Clear our internal halted state */
	1489	+ ep->ep_state &= ~EP_HALTED;
1202	1490
1203		- xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1204		- "Queueing configure endpoint command");
1205		- xhci_queue_configure_endpoint(xhci, command,
1206		- xhci->devs[slot_id]->in_ctx->dma, slot_id,
1207		- false);
1208		- xhci_ring_cmd_db(xhci);
1209		- } else {
1210		- /* Clear our internal halted state */
1211		- ep->ep_state &= ~EP_HALTED;
1212		- }
	1491	+ xhci_giveback_invalidated_tds(ep);
	1492	+
	1493	+ /* if this was a soft reset, then restart */
	1494	+ if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP)
	1495	+ ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1213	1496	}
1214	1497
1215	1498	static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
..	..	@@ -1239,7 +1522,7 @@
1239	1522	}
1240	1523
1241	1524	static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
1242		- struct xhci_event_cmd *event, u32 cmd_comp_code)
	1525	+ u32 cmd_comp_code)
1243	1526	{
1244	1527	struct xhci_virt_device *virt_dev;
1245	1528	struct xhci_input_control_ctx *ctrl_ctx;
..	..	@@ -1257,6 +1540,8 @@
1257	1540	* is not waiting on the configure endpoint command.
1258	1541	*/
1259	1542	virt_dev = xhci->devs[slot_id];
	1543	+ if (!virt_dev)
	1544	+ return;
1260	1545	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
1261	1546	if (!ctrl_ctx) {
1262	1547	xhci_warn(xhci, "Could not get input context, bad type.\n");
..	..	@@ -1301,24 +1586,27 @@
1301	1586	struct xhci_slot_ctx *slot_ctx;
1302	1587
1303	1588	vdev = xhci->devs[slot_id];
	1589	+ if (!vdev)
	1590	+ return;
1304	1591	slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1305	1592	trace_xhci_handle_cmd_addr_dev(slot_ctx);
1306	1593	}
1307	1594
1308		-static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id,
1309		- struct xhci_event_cmd *event)
	1595	+static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id)
1310	1596	{
1311	1597	struct xhci_virt_device *vdev;
1312	1598	struct xhci_slot_ctx *slot_ctx;
1313	1599
1314	1600	vdev = xhci->devs[slot_id];
	1601	+ if (!vdev) {
	1602	+ xhci_warn(xhci, "Reset device command completion for disabled slot %u\n",
	1603	+ slot_id);
	1604	+ return;
	1605	+ }
1315	1606	slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1316	1607	trace_xhci_handle_cmd_reset_dev(slot_ctx);
1317	1608
1318	1609	xhci_dbg(xhci, "Completed reset device command.\n");
1319		- if (!xhci->devs[slot_id])
1320		- xhci_warn(xhci, "Reset device command completion "
1321		- "for disabled slot %u\n", slot_id);
1322	1610	}
1323	1611
1324	1612	static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
..	..	@@ -1411,13 +1699,18 @@
1411	1699	static void handle_cmd_completion(struct xhci_hcd *xhci,
1412	1700	struct xhci_event_cmd *event)
1413	1701	{
1414		- int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
	1702	+ unsigned int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1415	1703	u64 cmd_dma;
1416	1704	dma_addr_t cmd_dequeue_dma;
1417	1705	u32 cmd_comp_code;
1418	1706	union xhci_trb *cmd_trb;
1419	1707	struct xhci_command *cmd;
1420	1708	u32 cmd_type;
	1709	+
	1710	+ if (slot_id >= MAX_HC_SLOTS) {
	1711	+ xhci_warn(xhci, "Invalid slot_id %u\n", slot_id);
	1712	+ return;
	1713	+ }
1421	1714
1422	1715	cmd_dma = le64_to_cpu(event->cmd_trb);
1423	1716	cmd_trb = xhci->cmd_ring->dequeue;
..	..	@@ -1479,8 +1772,7 @@
1479	1772	break;
1480	1773	case TRB_CONFIG_EP:
1481	1774	if (!cmd->completion)
1482		- xhci_handle_cmd_config_ep(xhci, slot_id, event,
1483		- cmd_comp_code);
	1775	+ xhci_handle_cmd_config_ep(xhci, slot_id, cmd_comp_code);
1484	1776	break;
1485	1777	case TRB_EVAL_CONTEXT:
1486	1778	break;
..	..	@@ -1491,7 +1783,8 @@
1491	1783	WARN_ON(slot_id != TRB_TO_SLOT_ID(
1492	1784	le32_to_cpu(cmd_trb->generic.field[3])));
1493	1785	if (!cmd->completion)
1494		- xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, event);
	1786	+ xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb,
	1787	+ cmd_comp_code);
1495	1788	break;
1496	1789	case TRB_SET_DEQ:
1497	1790	WARN_ON(slot_id != TRB_TO_SLOT_ID(
..	..	@@ -1514,7 +1807,7 @@
1514	1807	*/
1515	1808	slot_id = TRB_TO_SLOT_ID(
1516	1809	le32_to_cpu(cmd_trb->generic.field[3]));
1517		- xhci_handle_cmd_reset_dev(xhci, slot_id, event);
	1810	+ xhci_handle_cmd_reset_dev(xhci, slot_id);
1518	1811	break;
1519	1812	case TRB_NEC_GET_FW:
1520	1813	xhci_handle_cmd_nec_get_fw(xhci, event);
..	..	@@ -1541,11 +1834,8 @@
1541	1834	}
1542	1835
1543	1836	static void handle_vendor_event(struct xhci_hcd *xhci,
1544		- union xhci_trb *event)
	1837	+ union xhci_trb *event, u32 trb_type)
1545	1838	{
1546		- u32 trb_type;
1547		-
1548		- trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->generic.field[3]));
1549	1839	xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
1550	1840	if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
1551	1841	handle_cmd_completion(xhci, &event->event_cmd);
..	..	@@ -1619,18 +1909,19 @@
1619	1909	"WARN: xHC returned failed port status event\n");
1620	1910
1621	1911	port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
1622		- xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id);
1623		-
1624	1912	max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
	1913	+
1625	1914	if ((port_id <= 0) \|\| (port_id > max_ports)) {
1626		- xhci_warn(xhci, "Invalid port id %d\n", port_id);
	1915	+ xhci_warn(xhci, "Port change event with invalid port ID %d\n",
	1916	+ port_id);
1627	1917	inc_deq(xhci, xhci->event_ring);
1628	1918	return;
1629	1919	}
1630	1920
1631	1921	port = &xhci->hw_ports[port_id - 1];
1632	1922	if (!port \|\| !port->rhub \|\| port->hcd_portnum == DUPLICATE_ENTRY) {
1633		- xhci_warn(xhci, "Event for invalid port %u\n", port_id);
	1923	+ xhci_warn(xhci, "Port change event, no port for port ID %u\n",
	1924	+ port_id);
1634	1925	bogus_port_status = true;
1635	1926	goto cleanup;
1636	1927	}
..	..	@@ -1643,9 +1934,12 @@
1643	1934	}
1644	1935
1645	1936	hcd = port->rhub->hcd;
1646		- bus_state = &xhci->bus_state[hcd_index(hcd)];
	1937	+ bus_state = &port->rhub->bus_state;
1647	1938	hcd_portnum = port->hcd_portnum;
1648	1939	portsc = readl(port->addr);
	1940	+
	1941	+ xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n",
	1942	+ hcd->self.busnum, hcd_portnum + 1, port_id, portsc);
1649	1943
1650	1944	trace_xhci_handle_port_status(hcd_portnum, portsc);
1651	1945
..	..	@@ -1678,8 +1972,8 @@
1678	1972	*/
1679	1973	bus_state->port_remote_wakeup \|= 1 << hcd_portnum;
1680	1974	xhci_test_and_clear_bit(xhci, port, PORT_PLC);
1681		- xhci_set_link_state(xhci, port, XDEV_U0);
1682	1975	usb_hcd_start_port_resume(&hcd->self, hcd_portnum);
	1976	+ xhci_set_link_state(xhci, port, XDEV_U0);
1683	1977	/* Need to wait until the next link state change
1684	1978	* indicates the device is actually in U0.
1685	1979	*/
..	..	@@ -1708,6 +2002,7 @@
1708	2002	(portsc & PORT_PLS_MASK) == XDEV_U1 \|\|
1709	2003	(portsc & PORT_PLS_MASK) == XDEV_U2)) {
1710	2004	xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
	2005	+ complete(&bus_state->u3exit_done[hcd_portnum]);
1711	2006	/* We've just brought the device into U0/1/2 through either the
1712	2007	* Resume state after a device remote wakeup, or through the
1713	2008	* U3Exit state after a host-initiated resume. If it's a device
..	..	@@ -1765,7 +2060,8 @@
1765	2060	* bits are still set. When an event occurs, switch over to
1766	2061	* polling to avoid losing status changes.
1767	2062	*/
1768		- xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
	2063	+ xhci_dbg(xhci, "%s: starting usb%d port polling.\n",
	2064	+ __func__, hcd->self.busnum);
1769	2065	set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1770	2066	spin_unlock(&xhci->lock);
1771	2067	/* Pass this up to the core */
..	..	@@ -1840,34 +2136,21 @@
1840	2136	return NULL;
1841	2137	}
1842	2138
1843		-static void xhci_cleanup_halted_endpoint(struct xhci_hcd *xhci,
1844		- unsigned int slot_id, unsigned int ep_index,
1845		- unsigned int stream_id, struct xhci_td *td,
1846		- enum xhci_ep_reset_type reset_type)
	2139	+static void xhci_clear_hub_tt_buffer(struct xhci_hcd xhci, struct xhci_td td,
	2140	+ struct xhci_virt_ep *ep)
1847	2141	{
1848		- struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
1849		- struct xhci_command *command;
1850		-
1851	2142	/*
1852		- * Avoid resetting endpoint if link is inactive. Can cause host hang.
1853		- * Device will be reset soon to recover the link so don't do anything
	2143	+ * As part of low/full-speed endpoint-halt processing
	2144	+ * we must clear the TT buffer (USB 2.0 specification 11.17.5).
1854	2145	*/
1855		- if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR)
1856		- return;
1857		-
1858		- command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1859		- if (!command)
1860		- return;
1861		-
1862		- ep->ep_state \|= EP_HALTED;
1863		-
1864		- xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type);
1865		-
1866		- if (reset_type == EP_HARD_RESET) {
1867		- ep->ep_state \|= EP_HARD_CLEAR_TOGGLE;
1868		- xhci_cleanup_stalled_ring(xhci, ep_index, stream_id, td);
	2146	+ if (td->urb->dev->tt && !usb_pipeint(td->urb->pipe) &&
	2147	+ (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) &&
	2148	+ !(ep->ep_state & EP_CLEARING_TT)) {
	2149	+ ep->ep_state \|= EP_CLEARING_TT;
	2150	+ td->urb->ep->hcpriv = td->urb->dev;
	2151	+ if (usb_hub_clear_tt_buffer(td->urb))
	2152	+ ep->ep_state &= ~EP_CLEARING_TT;
1869	2153	}
1870		- xhci_ring_cmd_db(xhci);
1871	2154	}
1872	2155
1873	2156	/* Check if an error has halted the endpoint ring. The class driver will
..	..	@@ -1910,97 +2193,88 @@
1910	2193	return 0;
1911	2194	}
1912	2195
1913		-static int xhci_td_cleanup(struct xhci_hcd xhci, struct xhci_td td,
1914		- struct xhci_ring ep_ring, int status)
	2196	+static int finish_td(struct xhci_hcd xhci, struct xhci_virt_ep ep,
	2197	+ struct xhci_ring ep_ring, struct xhci_td td,
	2198	+ u32 trb_comp_code)
1915	2199	{
1916		- struct urb *urb = NULL;
1917		-
1918		- /* Clean up the endpoint's TD list */
1919		- urb = td->urb;
1920		-
1921		- /* if a bounce buffer was used to align this td then unmap it */
1922		- xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);
1923		-
1924		- /* Do one last check of the actual transfer length.
1925		- * If the host controller said we transferred more data than the buffer
1926		- * length, urb->actual_length will be a very big number (since it's
1927		- * unsigned). Play it safe and say we didn't transfer anything.
1928		- */
1929		- if (urb->actual_length > urb->transfer_buffer_length) {
1930		- xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
1931		- urb->transfer_buffer_length, urb->actual_length);
1932		- urb->actual_length = 0;
1933		- *status = 0;
1934		- }
1935		- list_del_init(&td->td_list);
1936		- /* Was this TD slated to be cancelled but completed anyway? */
1937		- if (!list_empty(&td->cancelled_td_list))
1938		- list_del_init(&td->cancelled_td_list);
1939		-
1940		- inc_td_cnt(urb);
1941		- /* Giveback the urb when all the tds are completed */
1942		- if (last_td_in_urb(td)) {
1943		- if ((urb->actual_length != urb->transfer_buffer_length &&
1944		- (urb->transfer_flags & URB_SHORT_NOT_OK)) \|\|
1945		- (*status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc)))
1946		- xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
1947		- urb, urb->actual_length,
1948		- urb->transfer_buffer_length, *status);
1949		-
1950		- /* set isoc urb status to 0 just as EHCI, UHCI, and OHCI */
1951		- if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
1952		- *status = 0;
1953		- xhci_giveback_urb_in_irq(xhci, td, *status);
1954		- }
1955		-
1956		- return 0;
1957		-}
1958		-
1959		-static int finish_td(struct xhci_hcd xhci, struct xhci_td td,
1960		- struct xhci_transfer_event *event,
1961		- struct xhci_virt_ep ep, int status)
1962		-{
1963		- struct xhci_virt_device *xdev;
1964	2200	struct xhci_ep_ctx *ep_ctx;
1965		- struct xhci_ring *ep_ring;
1966		- unsigned int slot_id;
1967		- u32 trb_comp_code;
1968		- int ep_index;
1969	2201
1970		- slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1971		- xdev = xhci->devs[slot_id];
1972		- ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1973		- ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1974		- ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1975		- trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
	2202	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index);
1976	2203
1977		- if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID \|\|
1978		- trb_comp_code == COMP_STOPPED \|\|
1979		- trb_comp_code == COMP_STOPPED_SHORT_PACKET) {
1980		- /* The Endpoint Stop Command completion will take care of any
1981		- * stopped TDs. A stopped TD may be restarted, so don't update
	2204	+ switch (trb_comp_code) {
	2205	+ case COMP_STOPPED_LENGTH_INVALID:
	2206	+ case COMP_STOPPED_SHORT_PACKET:
	2207	+ case COMP_STOPPED:
	2208	+ /*
	2209	+ * The "Stop Endpoint" completion will take care of any
	2210	+ * stopped TDs. A stopped TD may be restarted, so don't update
1982	2211	* the ring dequeue pointer or take this TD off any lists yet.
1983	2212	*/
1984	2213	return 0;
1985		- }
1986		- if (trb_comp_code == COMP_STALL_ERROR \|\|
1987		- xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
1988		- trb_comp_code)) {
1989		- /* Issue a reset endpoint command to clear the host side
1990		- * halt, followed by a set dequeue command to move the
1991		- * dequeue pointer past the TD.
1992		- * The class driver clears the device side halt later.
	2214	+ case COMP_USB_TRANSACTION_ERROR:
	2215	+ case COMP_BABBLE_DETECTED_ERROR:
	2216	+ case COMP_SPLIT_TRANSACTION_ERROR:
	2217	+ /*
	2218	+ * If endpoint context state is not halted we might be
	2219	+ * racing with a reset endpoint command issued by a unsuccessful
	2220	+ * stop endpoint completion (context error). In that case the
	2221	+ * td should be on the cancelled list, and EP_HALTED flag set.
	2222	+ *
	2223	+ * Or then it's not halted due to the 0.95 spec stating that a
	2224	+ * babbling control endpoint should not halt. The 0.96 spec
	2225	+ * again says it should. Some HW claims to be 0.95 compliant,
	2226	+ * but it halts the control endpoint anyway.
1993	2227	*/
1994		- xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
1995		- ep_ring->stream_id, td, EP_HARD_RESET);
1996		- } else {
1997		- /* Update ring dequeue pointer */
1998		- while (ep_ring->dequeue != td->last_trb)
1999		- inc_deq(xhci, ep_ring);
2000		- inc_deq(xhci, ep_ring);
	2228	+ if (GET_EP_CTX_STATE(ep_ctx) != EP_STATE_HALTED) {
	2229	+ /*
	2230	+ * If EP_HALTED is set and TD is on the cancelled list
	2231	+ * the TD and dequeue pointer will be handled by reset
	2232	+ * ep command completion
	2233	+ */
	2234	+ if ((ep->ep_state & EP_HALTED) &&
	2235	+ !list_empty(&td->cancelled_td_list)) {
	2236	+ xhci_dbg(xhci, "Already resolving halted ep for 0x%llx\n",
	2237	+ (unsigned long long)xhci_trb_virt_to_dma(
	2238	+ td->start_seg, td->first_trb));
	2239	+ return 0;
	2240	+ }
	2241	+ /* endpoint not halted, don't reset it */
	2242	+ break;
	2243	+ }
	2244	+ /* Almost same procedure as for STALL_ERROR below */
	2245	+ xhci_clear_hub_tt_buffer(xhci, td, ep);
	2246	+ xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,
	2247	+ EP_HARD_RESET);
	2248	+ return 0;
	2249	+ case COMP_STALL_ERROR:
	2250	+ /*
	2251	+ * xhci internal endpoint state will go to a "halt" state for
	2252	+ * any stall, including default control pipe protocol stall.
	2253	+ * To clear the host side halt we need to issue a reset endpoint
	2254	+ * command, followed by a set dequeue command to move past the
	2255	+ * TD.
	2256	+ * Class drivers clear the device side halt from a functional
	2257	+ * stall later. Hub TT buffer should only be cleared for FS/LS
	2258	+ * devices behind HS hubs for functional stalls.
	2259	+ */
	2260	+ if (ep->ep_index != 0)
	2261	+ xhci_clear_hub_tt_buffer(xhci, td, ep);
	2262	+
	2263	+ xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,
	2264	+ EP_HARD_RESET);
	2265	+
	2266	+ return 0; /* xhci_handle_halted_endpoint marked td cancelled */
	2267	+ default:
	2268	+ break;
2001	2269	}
2002	2270
2003		- return xhci_td_cleanup(xhci, td, ep_ring, status);
	2271	+ /* Update ring dequeue pointer */
	2272	+ ep_ring->dequeue = td->last_trb;
	2273	+ ep_ring->deq_seg = td->last_trb_seg;
	2274	+ ep_ring->num_trbs_free += td->num_trbs - 1;
	2275	+ inc_deq(xhci, ep_ring);
	2276	+
	2277	+ return xhci_td_cleanup(xhci, td, ep_ring, td->status);
2004	2278	}
2005	2279
2006	2280	/* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */
..	..	@@ -2021,23 +2295,17 @@
2021	2295	/*
2022	2296	* Process control tds, update urb status and actual_length.
2023	2297	*/
2024		-static int process_ctrl_td(struct xhci_hcd xhci, struct xhci_td td,
2025		- union xhci_trb ep_trb, struct xhci_transfer_event event,
2026		- struct xhci_virt_ep ep, int status)
	2298	+static int process_ctrl_td(struct xhci_hcd xhci, struct xhci_virt_ep ep,
	2299	+ struct xhci_ring ep_ring, struct xhci_td td,
	2300	+ union xhci_trb ep_trb, struct xhci_transfer_event event)
2027	2301	{
2028		- struct xhci_virt_device *xdev;
2029		- unsigned int slot_id;
2030		- int ep_index;
2031	2302	struct xhci_ep_ctx *ep_ctx;
2032	2303	u32 trb_comp_code;
2033	2304	u32 remaining, requested;
2034	2305	u32 trb_type;
2035	2306
2036	2307	trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3]));
2037		- slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
2038		- xdev = xhci->devs[slot_id];
2039		- ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
2040		- ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
	2308	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index);
2041	2309	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2042	2310	requested = td->urb->transfer_buffer_length;
2043	2311	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
..	..	@@ -2047,13 +2315,13 @@
2047	2315	if (trb_type != TRB_STATUS) {
2048	2316	xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n",
2049	2317	(trb_type == TRB_DATA) ? "data" : "setup");
2050		- *status = -ESHUTDOWN;
	2318	+ td->status = -ESHUTDOWN;
2051	2319	break;
2052	2320	}
2053		- *status = 0;
	2321	+ td->status = 0;
2054	2322	break;
2055	2323	case COMP_SHORT_PACKET:
2056		- *status = 0;
	2324	+ td->status = 0;
2057	2325	break;
2058	2326	case COMP_STOPPED_SHORT_PACKET:
2059	2327	if (trb_type == TRB_DATA \|\| trb_type == TRB_NORMAL)
..	..	@@ -2085,8 +2353,8 @@
2085	2353	ep_ctx, trb_comp_code))
2086	2354	break;
2087	2355	xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n",
2088		- trb_comp_code, ep_index);
2089		- /* else fall through */
	2356	+ trb_comp_code, ep->ep_index);
	2357	+ fallthrough;
2090	2358	case COMP_STALL_ERROR:
2091	2359	/* Did we transfer part of the data (middle) phase? */
2092	2360	if (trb_type == TRB_DATA \|\| trb_type == TRB_NORMAL)
..	..	@@ -2117,17 +2385,16 @@
2117	2385	td->urb->actual_length = requested;
2118	2386
2119	2387	finish_td:
2120		- return finish_td(xhci, td, event, ep, status);
	2388	+ return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2121	2389	}
2122	2390
2123	2391	/*
2124	2392	* Process isochronous tds, update urb packet status and actual_length.
2125	2393	*/
2126		-static int process_isoc_td(struct xhci_hcd xhci, struct xhci_td td,
2127		- union xhci_trb ep_trb, struct xhci_transfer_event event,
2128		- struct xhci_virt_ep ep, int status)
	2394	+static int process_isoc_td(struct xhci_hcd xhci, struct xhci_virt_ep ep,
	2395	+ struct xhci_ring ep_ring, struct xhci_td td,
	2396	+ union xhci_trb ep_trb, struct xhci_transfer_event event)
2129	2397	{
2130		- struct xhci_ring *ep_ring;
2131	2398	struct urb_priv *urb_priv;
2132	2399	int idx;
2133	2400	struct usb_iso_packet_descriptor *frame;
..	..	@@ -2136,7 +2403,6 @@
2136	2403	u32 remaining, requested, ep_trb_len;
2137	2404	int short_framestatus;
2138	2405
2139		- ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2140	2406	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2141	2407	urb_priv = td->urb->hcpriv;
2142	2408	idx = urb_priv->num_tds_done;
..	..	@@ -2197,26 +2463,23 @@
2197	2463	}
2198	2464
2199	2465	if (sum_trbs_for_length)
2200		- frame->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb) +
	2466	+ frame->actual_length = sum_trb_lengths(xhci, ep->ring, ep_trb) +
2201	2467	ep_trb_len - remaining;
2202	2468	else
2203	2469	frame->actual_length = requested;
2204	2470
2205	2471	td->urb->actual_length += frame->actual_length;
2206	2472
2207		- return finish_td(xhci, td, event, ep, status);
	2473	+ return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2208	2474	}
2209	2475
2210	2476	static int skip_isoc_td(struct xhci_hcd xhci, struct xhci_td td,
2211		- struct xhci_transfer_event *event,
2212		- struct xhci_virt_ep ep, int status)
	2477	+ struct xhci_virt_ep *ep, int status)
2213	2478	{
2214		- struct xhci_ring *ep_ring;
2215	2479	struct urb_priv *urb_priv;
2216	2480	struct usb_iso_packet_descriptor *frame;
2217	2481	int idx;
2218	2482
2219		- ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2220	2483	urb_priv = td->urb->hcpriv;
2221	2484	idx = urb_priv->num_tds_done;
2222	2485	frame = &td->urb->iso_frame_desc[idx];
..	..	@@ -2228,25 +2491,26 @@
2228	2491	frame->actual_length = 0;
2229	2492
2230	2493	/* Update ring dequeue pointer */
2231		- while (ep_ring->dequeue != td->last_trb)
2232		- inc_deq(xhci, ep_ring);
2233		- inc_deq(xhci, ep_ring);
	2494	+ ep->ring->dequeue = td->last_trb;
	2495	+ ep->ring->deq_seg = td->last_trb_seg;
	2496	+ ep->ring->num_trbs_free += td->num_trbs - 1;
	2497	+ inc_deq(xhci, ep->ring);
2234	2498
2235		- return xhci_td_cleanup(xhci, td, ep_ring, status);
	2499	+ return xhci_td_cleanup(xhci, td, ep->ring, status);
2236	2500	}
2237	2501
2238	2502	/*
2239	2503	* Process bulk and interrupt tds, update urb status and actual_length.
2240	2504	*/
2241		-static int process_bulk_intr_td(struct xhci_hcd xhci, struct xhci_td td,
2242		- union xhci_trb ep_trb, struct xhci_transfer_event event,
2243		- struct xhci_virt_ep ep, int status)
	2505	+static int process_bulk_intr_td(struct xhci_hcd xhci, struct xhci_virt_ep ep,
	2506	+ struct xhci_ring ep_ring, struct xhci_td td,
	2507	+ union xhci_trb ep_trb, struct xhci_transfer_event event)
2244	2508	{
2245		- struct xhci_ring *ep_ring;
	2509	+ struct xhci_slot_ctx *slot_ctx;
2246	2510	u32 trb_comp_code;
2247	2511	u32 remaining, requested, ep_trb_len;
2248	2512
2249		- ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
	2513	+ slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx);
2250	2514	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2251	2515	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2252	2516	ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
..	..	@@ -2254,6 +2518,7 @@
2254	2518
2255	2519	switch (trb_comp_code) {
2256	2520	case COMP_SUCCESS:
	2521	+ ep_ring->err_count = 0;
2257	2522	/* handle success with untransferred data as short packet */
2258	2523	if (ep_trb != td->last_trb \|\| remaining) {
2259	2524	xhci_warn(xhci, "WARN Successful completion on short TX\n");
..	..	@@ -2261,13 +2526,13 @@
2261	2526	td->urb->ep->desc.bEndpointAddress,
2262	2527	requested, remaining);
2263	2528	}
2264		- *status = 0;
	2529	+ td->status = 0;
2265	2530	break;
2266	2531	case COMP_SHORT_PACKET:
2267	2532	xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2268	2533	td->urb->ep->desc.bEndpointAddress,
2269	2534	requested, remaining);
2270		- *status = 0;
	2535	+ td->status = 0;
2271	2536	break;
2272	2537	case COMP_STOPPED_SHORT_PACKET:
2273	2538	td->urb->actual_length = remaining;
..	..	@@ -2277,6 +2542,17 @@
2277	2542	ep_trb_len = 0;
2278	2543	remaining = 0;
2279	2544	break;
	2545	+ case COMP_USB_TRANSACTION_ERROR:
	2546	+ if (xhci->quirks & XHCI_NO_SOFT_RETRY \|\|
	2547	+ (ep_ring->err_count++ > MAX_SOFT_RETRY) \|\|
	2548	+ le32_to_cpu(slot_ctx->tt_info) & TT_SLOT)
	2549	+ break;
	2550	+
	2551	+ td->status = 0;
	2552	+
	2553	+ xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td,
	2554	+ EP_SOFT_RESET);
	2555	+ return 0;
2280	2556	default:
2281	2557	/* do nothing */
2282	2558	break;
..	..	@@ -2294,7 +2570,8 @@
2294	2570	remaining);
2295	2571	td->urb->actual_length = 0;
2296	2572	}
2297		- return finish_td(xhci, td, event, ep, status);
	2573	+
	2574	+ return finish_td(xhci, ep, ep_ring, td, trb_comp_code);
2298	2575	}
2299	2576
2300	2577	/*
..	..	@@ -2305,7 +2582,6 @@
2305	2582	static int handle_tx_event(struct xhci_hcd *xhci,
2306	2583	struct xhci_transfer_event *event)
2307	2584	{
2308		- struct xhci_virt_device *xdev;
2309	2585	struct xhci_virt_ep *ep;
2310	2586	struct xhci_ring *ep_ring;
2311	2587	unsigned int slot_id;
..	..	@@ -2332,9 +2608,8 @@
2332	2608	goto err_out;
2333	2609	}
2334	2610
2335		- xdev = xhci->devs[slot_id];
2336	2611	ep_ring = xhci_dma_to_transfer_ring(ep, ep_trb_dma);
2337		- ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
	2612	+ ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index);
2338	2613
2339	2614	if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) {
2340	2615	xhci_err(xhci,
..	..	@@ -2350,8 +2625,8 @@
2350	2625	case COMP_USB_TRANSACTION_ERROR:
2351	2626	case COMP_INVALID_STREAM_TYPE_ERROR:
2352	2627	case COMP_INVALID_STREAM_ID_ERROR:
2353		- xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index, 0,
2354		- NULL, EP_SOFT_RESET);
	2628	+ xhci_handle_halted_endpoint(xhci, ep, 0, NULL,
	2629	+ EP_SOFT_RESET);
2355	2630	goto cleanup;
2356	2631	case COMP_RING_UNDERRUN:
2357	2632	case COMP_RING_OVERRUN:
..	..	@@ -2406,10 +2681,13 @@
2406	2681	case COMP_STALL_ERROR:
2407	2682	xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id,
2408	2683	ep_index);
2409		- ep->ep_state \|= EP_HALTED;
2410	2684	status = -EPIPE;
2411	2685	break;
2412	2686	case COMP_SPLIT_TRANSACTION_ERROR:
	2687	+ xhci_dbg(xhci, "Split transaction error for slot %u ep %u\n",
	2688	+ slot_id, ep_index);
	2689	+ status = -EPROTO;
	2690	+ break;
2413	2691	case COMP_USB_TRANSACTION_ERROR:
2414	2692	xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n",
2415	2693	slot_id, ep_index);
..	..	@@ -2527,6 +2805,14 @@
2527	2805	xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n",
2528	2806	slot_id, ep_index);
2529	2807	}
	2808	+ if (trb_comp_code == COMP_STALL_ERROR \|\|
	2809	+ xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
	2810	+ trb_comp_code)) {
	2811	+ xhci_handle_halted_endpoint(xhci, ep,
	2812	+ ep_ring->stream_id,
	2813	+ NULL,
	2814	+ EP_HARD_RESET);
	2815	+ }
2530	2816	goto cleanup;
2531	2817	}
2532	2818
..	..	@@ -2584,7 +2870,7 @@
2584	2870	return -ESHUTDOWN;
2585	2871	}
2586	2872
2587		- skip_isoc_td(xhci, td, event, ep, &status);
	2873	+ skip_isoc_td(xhci, td, ep, status);
2588	2874	goto cleanup;
2589	2875	}
2590	2876	if (trb_comp_code == COMP_SHORT_PACKET)
..	..	@@ -2612,25 +2898,26 @@
2612	2898	* endpoint. Otherwise, the endpoint remains stalled
2613	2899	* indefinitely.
2614	2900	*/
	2901	+
2615	2902	if (trb_is_noop(ep_trb)) {
2616	2903	if (trb_comp_code == COMP_STALL_ERROR \|\|
2617	2904	xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
2618	2905	trb_comp_code))
2619		- xhci_cleanup_halted_endpoint(xhci, slot_id,
2620		- ep_index,
2621		- ep_ring->stream_id,
2622		- td, EP_HARD_RESET);
	2906	+ xhci_handle_halted_endpoint(xhci, ep,
	2907	+ ep_ring->stream_id,
	2908	+ td, EP_HARD_RESET);
2623	2909	goto cleanup;
2624	2910	}
2625	2911
	2912	+ td->status = status;
	2913	+
2626	2914	/* update the urb's actual_length and give back to the core */
2627	2915	if (usb_endpoint_xfer_control(&td->urb->ep->desc))
2628		- process_ctrl_td(xhci, td, ep_trb, event, ep, &status);
	2916	+ process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event);
2629	2917	else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc))
2630		- process_isoc_td(xhci, td, ep_trb, event, ep, &status);
	2918	+ process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event);
2631	2919	else
2632		- process_bulk_intr_td(xhci, td, ep_trb, event, ep,
2633		- &status);
	2920	+ process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event);
2634	2921	cleanup:
2635	2922	handling_skipped_tds = ep->skip &&
2636	2923	trb_comp_code != COMP_MISSED_SERVICE_ERROR &&
..	..	@@ -2671,10 +2958,11 @@
2671	2958	* Returns >0 for "possibly more events to process" (caller should call again),
2672	2959	* otherwise 0 if done. In future, <0 returns should indicate error code.
2673	2960	*/
2674		-static int xhci_handle_event(struct xhci_hcd *xhci)
	2961	+int xhci_handle_event(struct xhci_hcd *xhci)
2675	2962	{
2676	2963	union xhci_trb *event;
2677	2964	int update_ptrs = 1;
	2965	+ u32 trb_type;
2678	2966	int ret;
2679	2967
2680	2968	/* Event ring hasn't been allocated yet. */
..	..	@@ -2696,31 +2984,30 @@
2696	2984	* speculative reads of the event's flags/data below.
2697	2985	*/
2698	2986	rmb();
	2987	+ trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->event_cmd.flags));
2699	2988	/* FIXME: Handle more event types. */
2700		- switch (le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) {
2701		- case TRB_TYPE(TRB_COMPLETION):
	2989	+
	2990	+ switch (trb_type) {
	2991	+ case TRB_COMPLETION:
2702	2992	handle_cmd_completion(xhci, &event->event_cmd);
2703	2993	break;
2704		- case TRB_TYPE(TRB_PORT_STATUS):
	2994	+ case TRB_PORT_STATUS:
2705	2995	handle_port_status(xhci, event);
2706	2996	update_ptrs = 0;
2707	2997	break;
2708		- case TRB_TYPE(TRB_TRANSFER):
	2998	+ case TRB_TRANSFER:
2709	2999	ret = handle_tx_event(xhci, &event->trans_event);
2710	3000	if (ret >= 0)
2711	3001	update_ptrs = 0;
2712	3002	break;
2713		- case TRB_TYPE(TRB_DEV_NOTE):
	3003	+ case TRB_DEV_NOTE:
2714	3004	handle_device_notification(xhci, event);
2715	3005	break;
2716	3006	default:
2717		- if ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) >=
2718		- TRB_TYPE(48))
2719		- handle_vendor_event(xhci, event);
	3007	+ if (trb_type >= TRB_VENDOR_DEFINED_LOW)
	3008	+ handle_vendor_event(xhci, event, trb_type);
2720	3009	else
2721		- xhci_warn(xhci, "ERROR unknown event type %d\n",
2722		- TRB_FIELD_TO_TYPE(
2723		- le32_to_cpu(event->event_cmd.flags)));
	3010	+ xhci_warn(xhci, "ERROR unknown event type %d\n", trb_type);
2724	3011	}
2725	3012	/* Any of the above functions may drop and re-acquire the lock, so check
2726	3013	* to make sure a watchdog timer didn't mark the host as non-responsive.
..	..	@@ -2740,13 +3027,14 @@
2740	3027	*/
2741	3028	return 1;
2742	3029	}
	3030	+EXPORT_SYMBOL_GPL(xhci_handle_event);
2743	3031
2744	3032	/*
2745	3033	* Update Event Ring Dequeue Pointer:
2746	3034	* - When all events have finished
2747	3035	* - To avoid "Event Ring Full Error" condition
2748	3036	*/
2749		-static void xhci_update_erst_dequeue(struct xhci_hcd *xhci,
	3037	+void xhci_update_erst_dequeue(struct xhci_hcd *xhci,
2750	3038	union xhci_trb *event_ring_deq)
2751	3039	{
2752	3040	u64 temp_64;
..	..	@@ -2775,6 +3063,16 @@
2775	3063	/* Clear the event handler busy flag (RW1C) */
2776	3064	temp_64 \|= ERST_EHB;
2777	3065	xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue);
	3066	+}
	3067	+EXPORT_SYMBOL_GPL(xhci_update_erst_dequeue);
	3068	+
	3069	+static irqreturn_t xhci_vendor_queue_irq_work(struct xhci_hcd *xhci)
	3070	+{
	3071	+ struct xhci_vendor_ops *ops = xhci_vendor_get_ops(xhci);
	3072	+
	3073	+ if (ops && ops->queue_irq_work)
	3074	+ return ops->queue_irq_work(xhci);
	3075	+ return IRQ_NONE;
2778	3076	}
2779	3077
2780	3078	/*
..	..	@@ -2810,6 +3108,10 @@
2810	3108	ret = IRQ_HANDLED;
2811	3109	goto out;
2812	3110	}
	3111	+
	3112	+ ret = xhci_vendor_queue_irq_work(xhci);
	3113	+ if (ret == IRQ_HANDLED)
	3114	+ goto out;
2813	3115
2814	3116	/*
2815	3117	* Clear the op reg interrupt status first,
..	..	@@ -2848,6 +3150,8 @@
2848	3150	if (event_loop++ < TRBS_PER_SEGMENT / 2)
2849	3151	continue;
2850	3152	xhci_update_erst_dequeue(xhci, event_ring_deq);
	3153	+ event_ring_deq = xhci->event_ring->dequeue;
	3154	+
2851	3155	event_loop = 0;
2852	3156	}
2853	3157
..	..	@@ -2901,6 +3205,7 @@
2901	3205	u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
2902	3206	{
2903	3207	unsigned int num_trbs_needed;
	3208	+ unsigned int link_trb_count = 0;
2904	3209
2905	3210	/* Make sure the endpoint has been added to xHC schedule */
2906	3211	switch (ep_state) {
..	..	@@ -2971,7 +3276,19 @@
2971	3276
2972	3277	ep_ring->enq_seg = ep_ring->enq_seg->next;
2973	3278	ep_ring->enqueue = ep_ring->enq_seg->trbs;
	3279	+
	3280	+ /* prevent infinite loop if all first trbs are link trbs */
	3281	+ if (link_trb_count++ > ep_ring->num_segs) {
	3282	+ xhci_warn(xhci, "Ring is an endless link TRB loop\n");
	3283	+ return -EINVAL;
	3284	+ }
2974	3285	}
	3286	+
	3287	+ if (last_trb_on_seg(ep_ring->enq_seg, ep_ring->enqueue)) {
	3288	+ xhci_warn(xhci, "Missing link TRB at end of ring segment\n");
	3289	+ return -EINVAL;
	3290	+ }
	3291	+
2975	3292	return 0;
2976	3293	}
2977	3294
..	..	@@ -2990,7 +3307,8 @@
2990	3307	struct xhci_ring *ep_ring;
2991	3308	struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2992	3309
2993		- ep_ring = xhci_stream_id_to_ring(xdev, ep_index, stream_id);
	3310	+ ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index,
	3311	+ stream_id);
2994	3312	if (!ep_ring) {
2995	3313	xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n",
2996	3314	stream_id);
..	..	@@ -3272,7 +3590,6 @@
3272	3590	bool more_trbs_coming = true;
3273	3591	bool need_zero_pkt = false;
3274	3592	bool first_trb = true;
3275		- bool en_trb_ent = true;
3276	3593	unsigned int num_trbs;
3277	3594	unsigned int start_cycle, num_sgs = 0;
3278	3595	unsigned int enqd_len, block_len, trb_buff_len, full_len;
..	..	@@ -3309,13 +3626,6 @@
3309	3626	if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1)
3310	3627	need_zero_pkt = true;
3311	3628
3312		- /*
3313		- * Don't enable the ENT flag in the TRB if
3314		- * the EP support bulk streaming protocol.
3315		- */
3316		- if (urb->stream_id)
3317		- en_trb_ent = false;
3318		-
3319	3629	td = &urb_priv->td[0];
3320	3630
3321	3631	/*
..	..	@@ -3347,22 +3657,11 @@
3347	3657	} else
3348	3658	field \|= ring->cycle_state;
3349	3659
3350		- /*
3351		- * Don't enable the ENT flag in the TRB if the
3352		- * transfer length of the TRB isn't an integer
3353		- * multiple of the EP maxpacket.
3354		- */
3355		- if (en_trb_ent &&
3356		- (trb_buff_len % usb_endpoint_maxp(&urb->ep->desc)))
3357		- en_trb_ent = false;
3358		-
3359	3660	/* Chain all the TRBs together; clear the chain bit in the last
3360	3661	* TRB to indicate it's the last TRB in the chain.
3361	3662	*/
3362	3663	if (enqd_len + trb_buff_len < full_len) {
3363	3664	field \|= TRB_CHAIN;
3364		- if (xhci->quirks & XHCI_TRB_ENT_QUIRK && en_trb_ent)
3365		- field \|= TRB_ENT;
3366	3665	if (trb_is_link(ring->enqueue + 1)) {
3367	3666	if (xhci_align_td(xhci, urb, enqd_len,
3368	3667	&trb_buff_len,
..	..	@@ -3378,6 +3677,13 @@
3378	3677	field \|= TRB_IOC;
3379	3678	more_trbs_coming = false;
3380	3679	td->last_trb = ring->enqueue;
	3680	+ td->last_trb_seg = ring->enq_seg;
	3681	+ if (xhci_urb_suitable_for_idt(urb)) {
	3682	+ memcpy(&send_addr, urb->transfer_buffer,
	3683	+ trb_buff_len);
	3684	+ le64_to_cpus(&send_addr);
	3685	+ field \|= TRB_IDT;
	3686	+ }
3381	3687	}
3382	3688
3383	3689	/* Only set interrupt on short packet for IN endpoints */
..	..	@@ -3397,7 +3703,7 @@
3397	3703	upper_32_bits(send_addr),
3398	3704	length_field,
3399	3705	field);
3400		-
	3706	+ td->num_trbs++;
3401	3707	addr += trb_buff_len;
3402	3708	sent_len = trb_buff_len;
3403	3709
..	..	@@ -3421,8 +3727,10 @@
3421	3727	ep_index, urb->stream_id,
3422	3728	1, urb, 1, mem_flags);
3423	3729	urb_priv->td[1].last_trb = ring->enqueue;
	3730	+ urb_priv->td[1].last_trb_seg = ring->enq_seg;
3424	3731	field = TRB_TYPE(TRB_NORMAL) \| ring->cycle_state \| TRB_IOC;
3425	3732	queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field);
	3733	+ urb_priv->td[1].num_trbs++;
3426	3734	}
3427	3735
3428	3736	check_trb_math(urb, enqd_len);
..	..	@@ -3473,6 +3781,7 @@
3473	3781
3474	3782	urb_priv = urb->hcpriv;
3475	3783	td = &urb_priv->td[0];
	3784	+ td->num_trbs = num_trbs;
3476	3785
3477	3786	/*
3478	3787	* Don't give the first TRB to the hardware (by toggling the cycle bit)
..	..	@@ -3516,6 +3825,16 @@
3516	3825
3517	3826	if (urb->transfer_buffer_length > 0) {
3518	3827	u32 length_field, remainder;
	3828	+ u64 addr;
	3829	+
	3830	+ if (xhci_urb_suitable_for_idt(urb)) {
	3831	+ memcpy(&addr, urb->transfer_buffer,
	3832	+ urb->transfer_buffer_length);
	3833	+ le64_to_cpus(&addr);
	3834	+ field \|= TRB_IDT;
	3835	+ } else {
	3836	+ addr = (u64) urb->transfer_dma;
	3837	+ }
3519	3838
3520	3839	remainder = xhci_td_remainder(xhci, 0,
3521	3840	urb->transfer_buffer_length,
..	..	@@ -3527,14 +3846,15 @@
3527	3846	if (setup->bRequestType & USB_DIR_IN)
3528	3847	field \|= TRB_DIR_IN;
3529	3848	queue_trb(xhci, ep_ring, true,
3530		- lower_32_bits(urb->transfer_dma),
3531		- upper_32_bits(urb->transfer_dma),
	3849	+ lower_32_bits(addr),
	3850	+ upper_32_bits(addr),
3532	3851	length_field,
3533	3852	field \| ep_ring->cycle_state);
3534	3853	}
3535	3854
3536	3855	/* Save the DMA address of the last TRB in the TD */
3537	3856	td->last_trb = ep_ring->enqueue;
	3857	+ td->last_trb_seg = ep_ring->enq_seg;
3538	3858
3539	3859	/* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */
3540	3860	/* If the device sent data, the status stage is an OUT transfer */
..	..	@@ -3698,6 +4018,24 @@
3698	4018	return start_frame;
3699	4019	}
3700	4020
	4021	+/* Check if we should generate event interrupt for a TD in an isoc URB */
	4022	+static bool trb_block_event_intr(struct xhci_hcd *xhci, int num_tds, int i)
	4023	+{
	4024	+ if (xhci->hci_version < 0x100)
	4025	+ return false;
	4026	+ /* always generate an event interrupt for the last TD */
	4027	+ if (i == num_tds - 1)
	4028	+ return false;
	4029	+ /*
	4030	+ * If AVOID_BEI is set the host handles full event rings poorly,
	4031	+ * generate an event at least every 8th TD to clear the event ring
	4032	+ */
	4033	+ if (i && xhci->quirks & XHCI_AVOID_BEI)
	4034	+ return !!(i % 8);
	4035	+
	4036	+ return true;
	4037	+}
	4038	+
3701	4039	/* This is for isoc transfer */
3702	4040	static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3703	4041	struct urb *urb, int slot_id, unsigned int ep_index)
..	..	@@ -3761,7 +4099,7 @@
3761	4099	goto cleanup;
3762	4100	}
3763	4101	td = &urb_priv->td[i];
3764		-
	4102	+ td->num_trbs = trbs_per_td;
3765	4103	/* use SIA as default, if frame id is used overwrite it */
3766	4104	sia_frame_id = TRB_SIA;
3767	4105	if (!(urb->transfer_flags & URB_ISO_ASAP) &&
..	..	@@ -3804,11 +4142,9 @@
3804	4142	} else {
3805	4143	more_trbs_coming = false;
3806	4144	td->last_trb = ep_ring->enqueue;
	4145	+ td->last_trb_seg = ep_ring->enq_seg;
3807	4146	field \|= TRB_IOC;
3808		- /* set BEI, except for the last TD */
3809		- if (xhci->hci_version >= 0x100 &&
3810		- !(xhci->quirks & XHCI_AVOID_BEI) &&
3811		- i < num_tds - 1)
	4147	+ if (trb_block_event_intr(xhci, num_tds, i))
3812	4148	field \|= TRB_BEI;
3813	4149	}
3814	4150	/* Calculate TRB length */
..	..	@@ -4089,71 +4425,7 @@
4089	4425	return queue_command(xhci, cmd, 0, 0, 0,
4090	4426	trb_slot_id \| trb_ep_index \| type \| trb_suspend, false);
4091	4427	}
4092		-
4093		-/* Set Transfer Ring Dequeue Pointer command */
4094		-void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci,
4095		- unsigned int slot_id, unsigned int ep_index,
4096		- struct xhci_dequeue_state *deq_state)
4097		-{
4098		- dma_addr_t addr;
4099		- u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
4100		- u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
4101		- u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id);
4102		- u32 trb_sct = 0;
4103		- u32 type = TRB_TYPE(TRB_SET_DEQ);
4104		- struct xhci_virt_ep *ep;
4105		- struct xhci_command *cmd;
4106		- int ret;
4107		-
4108		- xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
4109		- "Set TR Deq Ptr cmd, new deq seg = %p (0x%llx dma), new deq ptr = %p (0x%llx dma), new cycle = %u",
4110		- deq_state->new_deq_seg,
4111		- (unsigned long long)deq_state->new_deq_seg->dma,
4112		- deq_state->new_deq_ptr,
4113		- (unsigned long long)xhci_trb_virt_to_dma(
4114		- deq_state->new_deq_seg, deq_state->new_deq_ptr),
4115		- deq_state->new_cycle_state);
4116		-
4117		- addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
4118		- deq_state->new_deq_ptr);
4119		- if (addr == 0) {
4120		- xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
4121		- xhci_warn(xhci, "WARN deq seg = %p, deq pt = %p\n",
4122		- deq_state->new_deq_seg, deq_state->new_deq_ptr);
4123		- return;
4124		- }
4125		- ep = &xhci->devs[slot_id]->eps[ep_index];
4126		- if ((ep->ep_state & SET_DEQ_PENDING)) {
4127		- xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
4128		- xhci_warn(xhci, "A Set TR Deq Ptr command is pending.\n");
4129		- return;
4130		- }
4131		-
4132		- /* This function gets called from contexts where it cannot sleep */
4133		- cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC);
4134		- if (!cmd)
4135		- return;
4136		-
4137		- ep->queued_deq_seg = deq_state->new_deq_seg;
4138		- ep->queued_deq_ptr = deq_state->new_deq_ptr;
4139		- if (deq_state->stream_id)
4140		- trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
4141		- ret = queue_command(xhci, cmd,
4142		- lower_32_bits(addr) \| trb_sct \| deq_state->new_cycle_state,
4143		- upper_32_bits(addr), trb_stream_id,
4144		- trb_slot_id \| trb_ep_index \| type, false);
4145		- if (ret < 0) {
4146		- xhci_free_command(xhci, cmd);
4147		- return;
4148		- }
4149		-
4150		- /* Stop the TD queueing code from ringing the doorbell until
4151		- * this command completes. The HC won't set the dequeue pointer
4152		- * if the ring is running, and ringing the doorbell starts the
4153		- * ring running.
4154		- */
4155		- ep->ep_state \|= SET_DEQ_PENDING;
4156		-}
	4428	+EXPORT_SYMBOL_GPL(xhci_queue_stop_endpoint);
4157	4429
4158	4430	int xhci_queue_reset_ep(struct xhci_hcd xhci, struct xhci_command cmd,
4159	4431	int slot_id, unsigned int ep_index,