forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/drivers/net/wireless/mediatek/mt76/usb.c
....@@ -1,19 +1,9 @@
1
+// SPDX-License-Identifier: ISC
12 /*
23  * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>
3
- *
4
- * Permission to use, copy, modify, and/or distribute this software for any
5
- * purpose with or without fee is hereby granted, provided that the above
6
- * copyright notice and this permission notice appear in all copies.
7
- *
8
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
154  */
165 
6
+#include <linux/module.h>
177 #include "mt76.h"
188 #include "usb_trace.h"
199 #include "dma.h"
....@@ -21,26 +11,31 @@
2111 #define MT_VEND_REQ_MAX_RETRY	10
2212 #define MT_VEND_REQ_TOUT_MS	300
2313 
24
-/* should be called with usb_ctrl_mtx locked */
14
+static bool disable_usb_sg;
15
+module_param_named(disable_usb_sg, disable_usb_sg, bool, 0644);
16
+MODULE_PARM_DESC(disable_usb_sg, "Disable usb scatter-gather support");
17
+
2518 static int __mt76u_vendor_request(struct mt76_dev *dev, u8 req,
2619 				  u8 req_type, u16 val, u16 offset,
2720 				  void *buf, size_t len)
2821 {
29
-	struct usb_interface *intf = to_usb_interface(dev->dev);
30
-	struct usb_device *udev = interface_to_usbdev(intf);
22
+	struct usb_interface *uintf = to_usb_interface(dev->dev);
23
+	struct usb_device *udev = interface_to_usbdev(uintf);
3124 	unsigned int pipe;
3225 	int i, ret;
26
+
27
+	lockdep_assert_held(&dev->usb.usb_ctrl_mtx);
3328 
3429 	pipe = (req_type & USB_DIR_IN) ? usb_rcvctrlpipe(udev, 0)
3530 				       : usb_sndctrlpipe(udev, 0);
3631 	for (i = 0; i < MT_VEND_REQ_MAX_RETRY; i++) {
37
-		if (test_bit(MT76_REMOVED, &dev->state))
32
+		if (test_bit(MT76_REMOVED, &dev->phy.state))
3833 			return -EIO;
3934 
4035 		ret = usb_control_msg(udev, pipe, req, req_type, val,
4136 				      offset, buf, len, MT_VEND_REQ_TOUT_MS);
4237 		if (ret == -ENODEV)
43
-			set_bit(MT76_REMOVED, &dev->state);
38
+			set_bit(MT76_REMOVED, &dev->phy.state);
4439 		if (ret >= 0 || ret == -ENODEV)
4540 			return ret;
4641 		usleep_range(5000, 10000);
....@@ -67,13 +62,25 @@
6762 }
6863 EXPORT_SYMBOL_GPL(mt76u_vendor_request);
6964 
70
-/* should be called with usb_ctrl_mtx locked */
71
-static u32 __mt76u_rr(struct mt76_dev *dev, u32 addr)
65
+static u32 ___mt76u_rr(struct mt76_dev *dev, u8 req, u32 addr)
7266 {
7367 	struct mt76_usb *usb = &dev->usb;
7468 	u32 data = ~0;
75
-	u16 offset;
7669 	int ret;
70
+
71
+	ret = __mt76u_vendor_request(dev, req,
72
+				     USB_DIR_IN | USB_TYPE_VENDOR,
73
+				     addr >> 16, addr, usb->data,
74
+				     sizeof(__le32));
75
+	if (ret == sizeof(__le32))
76
+		data = get_unaligned_le32(usb->data);
77
+	trace_usb_reg_rr(dev, addr, data);
78
+
79
+	return data;
80
+}
81
+
82
+static u32 __mt76u_rr(struct mt76_dev *dev, u32 addr)
83
+{
7784 	u8 req;
7885 
7986 	switch (addr & MT_VEND_TYPE_MASK) {
....@@ -87,19 +94,11 @@
8794 		req = MT_VEND_MULTI_READ;
8895 		break;
8996 	}
90
-	offset = addr & ~MT_VEND_TYPE_MASK;
9197 
92
-	ret = __mt76u_vendor_request(dev, req,
93
-				     USB_DIR_IN | USB_TYPE_VENDOR,
94
-				     0, offset, usb->data, sizeof(__le32));
95
-	if (ret == sizeof(__le32))
96
-		data = get_unaligned_le32(usb->data);
97
-	trace_usb_reg_rr(dev, addr, data);
98
-
99
-	return data;
98
+	return ___mt76u_rr(dev, req, addr & ~MT_VEND_TYPE_MASK);
10099 }
101100 
102
-u32 mt76u_rr(struct mt76_dev *dev, u32 addr)
101
+static u32 mt76u_rr(struct mt76_dev *dev, u32 addr)
103102 {
104103 	u32 ret;
105104 
....@@ -110,11 +109,32 @@
110109 	return ret;
111110 }
112111 
113
-/* should be called with usb_ctrl_mtx locked */
114
-static void __mt76u_wr(struct mt76_dev *dev, u32 addr, u32 val)
112
+static u32 mt76u_rr_ext(struct mt76_dev *dev, u32 addr)
113
+{
114
+	u32 ret;
115
+
116
+	mutex_lock(&dev->usb.usb_ctrl_mtx);
117
+	ret = ___mt76u_rr(dev, MT_VEND_READ_EXT, addr);
118
+	mutex_unlock(&dev->usb.usb_ctrl_mtx);
119
+
120
+	return ret;
121
+}
122
+
123
+static void ___mt76u_wr(struct mt76_dev *dev, u8 req,
124
+			u32 addr, u32 val)
115125 {
116126 	struct mt76_usb *usb = &dev->usb;
117
-	u16 offset;
127
+
128
+	put_unaligned_le32(val, usb->data);
129
+	__mt76u_vendor_request(dev, req,
130
+			       USB_DIR_OUT | USB_TYPE_VENDOR,
131
+			       addr >> 16, addr, usb->data,
132
+			       sizeof(__le32));
133
+	trace_usb_reg_wr(dev, addr, val);
134
+}
135
+
136
+static void __mt76u_wr(struct mt76_dev *dev, u32 addr, u32 val)
137
+{
118138 	u8 req;
119139 
120140 	switch (addr & MT_VEND_TYPE_MASK) {
....@@ -125,19 +145,20 @@
125145 		req = MT_VEND_MULTI_WRITE;
126146 		break;
127147 	}
128
-	offset = addr & ~MT_VEND_TYPE_MASK;
129
-
130
-	put_unaligned_le32(val, usb->data);
131
-	__mt76u_vendor_request(dev, req,
132
-			       USB_DIR_OUT | USB_TYPE_VENDOR, 0,
133
-			       offset, usb->data, sizeof(__le32));
134
-	trace_usb_reg_wr(dev, addr, val);
148
+	___mt76u_wr(dev, req, addr & ~MT_VEND_TYPE_MASK, val);
135149 }
136150 
137
-void mt76u_wr(struct mt76_dev *dev, u32 addr, u32 val)
151
+static void mt76u_wr(struct mt76_dev *dev, u32 addr, u32 val)
138152 {
139153 	mutex_lock(&dev->usb.usb_ctrl_mtx);
140154 	__mt76u_wr(dev, addr, val);
155
+	mutex_unlock(&dev->usb.usb_ctrl_mtx);
156
+}
157
+
158
+static void mt76u_wr_ext(struct mt76_dev *dev, u32 addr, u32 val)
159
+{
160
+	mutex_lock(&dev->usb.usb_ctrl_mtx);
161
+	___mt76u_wr(dev, MT_VEND_WRITE_EXT, addr, val);
141162 	mutex_unlock(&dev->usb.usb_ctrl_mtx);
142163 }
143164 
....@@ -152,22 +173,94 @@
152173 	return val;
153174 }
154175 
176
+static u32 mt76u_rmw_ext(struct mt76_dev *dev, u32 addr,
177
+			 u32 mask, u32 val)
178
+{
179
+	mutex_lock(&dev->usb.usb_ctrl_mtx);
180
+	val |= ___mt76u_rr(dev, MT_VEND_READ_EXT, addr) & ~mask;
181
+	___mt76u_wr(dev, MT_VEND_WRITE_EXT, addr, val);
182
+	mutex_unlock(&dev->usb.usb_ctrl_mtx);
183
+
184
+	return val;
185
+}
186
+
155187 static void mt76u_copy(struct mt76_dev *dev, u32 offset,
156188 		       const void *data, int len)
157189 {
158190 	struct mt76_usb *usb = &dev->usb;
159
-	const u32 *val = data;
160
-	int i, ret;
191
+	const u8 *val = data;
192
+	int ret;
193
+	int current_batch_size;
194
+	int i = 0;
195
+
196
+	/* Assure that always a multiple of 4 bytes are copied,
197
+	 * otherwise beacons can be corrupted.
198
+	 * See: "mt76: round up length on mt76_wr_copy"
199
+	 * Commit 850e8f6fbd5d0003b0
200
+	 */
201
+	len = round_up(len, 4);
161202 
162203 	mutex_lock(&usb->usb_ctrl_mtx);
163
-	for (i = 0; i < (len / 4); i++) {
164
-		put_unaligned_le32(val[i], usb->data);
204
+	while (i < len) {
205
+		current_batch_size = min_t(int, usb->data_len, len - i);
206
+		memcpy(usb->data, val + i, current_batch_size);
165207 		ret = __mt76u_vendor_request(dev, MT_VEND_MULTI_WRITE,
166208 					     USB_DIR_OUT | USB_TYPE_VENDOR,
167
-					     0, offset + i * 4, usb->data,
168
-					     sizeof(__le32));
209
+					     0, offset + i, usb->data,
210
+					     current_batch_size);
169211 		if (ret < 0)
170212 			break;
213
+
214
+		i += current_batch_size;
215
+	}
216
+	mutex_unlock(&usb->usb_ctrl_mtx);
217
+}
218
+
219
+static void mt76u_copy_ext(struct mt76_dev *dev, u32 offset,
220
+			   const void *data, int len)
221
+{
222
+	struct mt76_usb *usb = &dev->usb;
223
+	int ret, i = 0, batch_len;
224
+	const u8 *val = data;
225
+
226
+	len = round_up(len, 4);
227
+	mutex_lock(&usb->usb_ctrl_mtx);
228
+	while (i < len) {
229
+		batch_len = min_t(int, usb->data_len, len - i);
230
+		memcpy(usb->data, val + i, batch_len);
231
+		ret = __mt76u_vendor_request(dev, MT_VEND_WRITE_EXT,
232
+					     USB_DIR_OUT | USB_TYPE_VENDOR,
233
+					     (offset + i) >> 16, offset + i,
234
+					     usb->data, batch_len);
235
+		if (ret < 0)
236
+			break;
237
+
238
+		i += batch_len;
239
+	}
240
+	mutex_unlock(&usb->usb_ctrl_mtx);
241
+}
242
+
243
+static void
244
+mt76u_read_copy_ext(struct mt76_dev *dev, u32 offset,
245
+		    void *data, int len)
246
+{
247
+	struct mt76_usb *usb = &dev->usb;
248
+	int i = 0, batch_len, ret;
249
+	u8 *val = data;
250
+
251
+	len = round_up(len, 4);
252
+	mutex_lock(&usb->usb_ctrl_mtx);
253
+	while (i < len) {
254
+		batch_len = min_t(int, usb->data_len, len - i);
255
+		ret = __mt76u_vendor_request(dev, MT_VEND_READ_EXT,
256
+					     USB_DIR_IN | USB_TYPE_VENDOR,
257
+					     (offset + i) >> 16, offset + i,
258
+					     usb->data, batch_len);
259
+		if (ret < 0)
260
+			break;
261
+
262
+		memcpy(val + i, usb->data, batch_len);
263
+		i += batch_len;
171264 	}
172265 	mutex_unlock(&usb->usb_ctrl_mtx);
173266 }
....@@ -187,6 +280,70 @@
187280 EXPORT_SYMBOL_GPL(mt76u_single_wr);
188281 
189282 static int
283
+mt76u_req_wr_rp(struct mt76_dev *dev, u32 base,
284
+		const struct mt76_reg_pair *data, int len)
285
+{
286
+	struct mt76_usb *usb = &dev->usb;
287
+
288
+	mutex_lock(&usb->usb_ctrl_mtx);
289
+	while (len > 0) {
290
+		__mt76u_wr(dev, base + data->reg, data->value);
291
+		len--;
292
+		data++;
293
+	}
294
+	mutex_unlock(&usb->usb_ctrl_mtx);
295
+
296
+	return 0;
297
+}
298
+
299
+static int
300
+mt76u_wr_rp(struct mt76_dev *dev, u32 base,
301
+	    const struct mt76_reg_pair *data, int n)
302
+{
303
+	if (test_bit(MT76_STATE_MCU_RUNNING, &dev->phy.state))
304
+		return dev->mcu_ops->mcu_wr_rp(dev, base, data, n);
305
+	else
306
+		return mt76u_req_wr_rp(dev, base, data, n);
307
+}
308
+
309
+static int
310
+mt76u_req_rd_rp(struct mt76_dev *dev, u32 base, struct mt76_reg_pair *data,
311
+		int len)
312
+{
313
+	struct mt76_usb *usb = &dev->usb;
314
+
315
+	mutex_lock(&usb->usb_ctrl_mtx);
316
+	while (len > 0) {
317
+		data->value = __mt76u_rr(dev, base + data->reg);
318
+		len--;
319
+		data++;
320
+	}
321
+	mutex_unlock(&usb->usb_ctrl_mtx);
322
+
323
+	return 0;
324
+}
325
+
326
+static int
327
+mt76u_rd_rp(struct mt76_dev *dev, u32 base,
328
+	    struct mt76_reg_pair *data, int n)
329
+{
330
+	if (test_bit(MT76_STATE_MCU_RUNNING, &dev->phy.state))
331
+		return dev->mcu_ops->mcu_rd_rp(dev, base, data, n);
332
+	else
333
+		return mt76u_req_rd_rp(dev, base, data, n);
334
+}
335
+
336
+static bool mt76u_check_sg(struct mt76_dev *dev)
337
+{
338
+	struct usb_interface *uintf = to_usb_interface(dev->dev);
339
+	struct usb_device *udev = interface_to_usbdev(uintf);
340
+
341
+	return (!disable_usb_sg && udev->bus->sg_tablesize > 0 &&
342
+		(udev->bus->no_sg_constraint ||
343
+		 udev->speed == USB_SPEED_WIRELESS));
344
+}
345
+
346
+static int
190347 mt76u_set_endpoints(struct usb_interface *intf,
191348 		    struct mt76_usb *usb)
192349 {
....@@ -200,12 +357,10 @@
200357 		if (usb_endpoint_is_bulk_in(ep_desc) &&
201358 		    in_ep < __MT_EP_IN_MAX) {
202359 			usb->in_ep[in_ep] = usb_endpoint_num(ep_desc);
203
-			usb->in_max_packet = usb_endpoint_maxp(ep_desc);
204360 			in_ep++;
205361 		} else if (usb_endpoint_is_bulk_out(ep_desc) &&
206362 			   out_ep < __MT_EP_OUT_MAX) {
207363 			usb->out_ep[out_ep] = usb_endpoint_num(ep_desc);
208
-			usb->out_max_packet = usb_endpoint_maxp(ep_desc);
209364 			out_ep++;
210365 		}
211366 	}
....@@ -216,10 +371,9 @@
216371 }
217372 
218373 static int
219
-mt76u_fill_rx_sg(struct mt76_dev *dev, struct mt76u_buf *buf,
220
-		 int nsgs, int len, int sglen)
374
+mt76u_fill_rx_sg(struct mt76_dev *dev, struct mt76_queue *q, struct urb *urb,
375
+		 int nsgs, gfp_t gfp)
221376 {
222
-	struct urb *urb = buf->urb;
223377 	int i;
224378 
225379 	for (i = 0; i < nsgs; i++) {
....@@ -227,13 +381,13 @@
227381 		void *data;
228382 		int offset;
229383 
230
-		data = netdev_alloc_frag(len);
384
+		data = page_frag_alloc(&q->rx_page, q->buf_size, gfp);
231385 		if (!data)
232386 			break;
233387 
234388 		page = virt_to_head_page(data);
235389 		offset = data - page_address(page);
236
-		sg_set_page(&urb->sg[i], page, sglen, offset);
390
+		sg_set_page(&urb->sg[i], page, q->buf_size, offset);
237391 	}
238392 
239393 	if (i < nsgs) {
....@@ -245,54 +399,83 @@
245399 	}
246400 
247401 	urb->num_sgs = max_t(int, i, urb->num_sgs);
248
-	buf->len = urb->num_sgs * sglen,
402
+	urb->transfer_buffer_length = urb->num_sgs * q->buf_size;
249403 	sg_init_marker(urb->sg, urb->num_sgs);
250404 
251405 	return i ? : -ENOMEM;
252406 }
253407 
254
-int mt76u_buf_alloc(struct mt76_dev *dev, struct mt76u_buf *buf,
255
-		    int nsgs, int len, int sglen, gfp_t gfp)
408
+static int
409
+mt76u_refill_rx(struct mt76_dev *dev, struct mt76_queue *q,
410
+		struct urb *urb, int nsgs, gfp_t gfp)
256411 {
257
-	buf->urb = usb_alloc_urb(0, gfp);
258
-	if (!buf->urb)
259
-		return -ENOMEM;
412
+	enum mt76_rxq_id qid = q - &dev->q_rx[MT_RXQ_MAIN];
260413 
261
-	buf->urb->sg = devm_kcalloc(dev->dev, nsgs, sizeof(*buf->urb->sg),
262
-				    gfp);
263
-	if (!buf->urb->sg)
264
-		return -ENOMEM;
414
+	if (qid == MT_RXQ_MAIN && dev->usb.sg_en)
415
+		return mt76u_fill_rx_sg(dev, q, urb, nsgs, gfp);
265416 
266
-	sg_init_table(buf->urb->sg, nsgs);
267
-	buf->dev = dev;
417
+	urb->transfer_buffer_length = q->buf_size;
418
+	urb->transfer_buffer = page_frag_alloc(&q->rx_page, q->buf_size, gfp);
268419 
269
-	return mt76u_fill_rx_sg(dev, buf, nsgs, len, sglen);
420
+	return urb->transfer_buffer ? 0 : -ENOMEM;
270421 }
271
-EXPORT_SYMBOL_GPL(mt76u_buf_alloc);
272422 
273
-void mt76u_buf_free(struct mt76u_buf *buf)
423
+static int
424
+mt76u_urb_alloc(struct mt76_dev *dev, struct mt76_queue_entry *e,
425
+		int sg_max_size)
274426 {
275
-	struct urb *urb = buf->urb;
276
-	struct scatterlist *sg;
427
+	unsigned int size = sizeof(struct urb);
428
+
429
+	if (dev->usb.sg_en)
430
+		size += sg_max_size * sizeof(struct scatterlist);
431
+
432
+	e->urb = kzalloc(size, GFP_KERNEL);
433
+	if (!e->urb)
434
+		return -ENOMEM;
435
+
436
+	usb_init_urb(e->urb);
437
+
438
+	if (dev->usb.sg_en && sg_max_size > 0)
439
+		e->urb->sg = (struct scatterlist *)(e->urb + 1);
440
+
441
+	return 0;
442
+}
443
+
444
+static int
445
+mt76u_rx_urb_alloc(struct mt76_dev *dev, struct mt76_queue *q,
446
+		   struct mt76_queue_entry *e)
447
+{
448
+	enum mt76_rxq_id qid = q - &dev->q_rx[MT_RXQ_MAIN];
449
+	int err, sg_size;
450
+
451
+	sg_size = qid == MT_RXQ_MAIN ? MT_RX_SG_MAX_SIZE : 0;
452
+	err = mt76u_urb_alloc(dev, e, sg_size);
453
+	if (err)
454
+		return err;
455
+
456
+	return mt76u_refill_rx(dev, q, e->urb, sg_size, GFP_KERNEL);
457
+}
458
+
459
+static void mt76u_urb_free(struct urb *urb)
460
+{
277461 	int i;
278462 
279
-	for (i = 0; i < urb->num_sgs; i++) {
280
-		sg = &urb->sg[i];
281
-		if (!sg)
282
-			continue;
463
+	for (i = 0; i < urb->num_sgs; i++)
464
+		skb_free_frag(sg_virt(&urb->sg[i]));
283465 
284
-		skb_free_frag(sg_virt(sg));
285
-	}
286
-	usb_free_urb(buf->urb);
466
+	if (urb->transfer_buffer)
467
+		skb_free_frag(urb->transfer_buffer);
468
+
469
+	usb_free_urb(urb);
287470 }
288
-EXPORT_SYMBOL_GPL(mt76u_buf_free);
289471 
290
-int mt76u_submit_buf(struct mt76_dev *dev, int dir, int index,
291
-		     struct mt76u_buf *buf, gfp_t gfp,
292
-		     usb_complete_t complete_fn, void *context)
472
+static void
473
+mt76u_fill_bulk_urb(struct mt76_dev *dev, int dir, int index,
474
+		    struct urb *urb, usb_complete_t complete_fn,
475
+		    void *context)
293476 {
294
-	struct usb_interface *intf = to_usb_interface(dev->dev);
295
-	struct usb_device *udev = interface_to_usbdev(intf);
477
+	struct usb_interface *uintf = to_usb_interface(dev->dev);
478
+	struct usb_device *udev = interface_to_usbdev(uintf);
296479 	unsigned int pipe;
297480 
298481 	if (dir == USB_DIR_IN)
....@@ -300,80 +483,115 @@
300483 	else
301484 		pipe = usb_sndbulkpipe(udev, dev->usb.out_ep[index]);
302485 
303
-	usb_fill_bulk_urb(buf->urb, udev, pipe, NULL, buf->len,
304
-			  complete_fn, context);
305
-
306
-	return usb_submit_urb(buf->urb, gfp);
486
+	urb->dev = udev;
487
+	urb->pipe = pipe;
488
+	urb->complete = complete_fn;
489
+	urb->context = context;
307490 }
308
-EXPORT_SYMBOL_GPL(mt76u_submit_buf);
309491 
310
-static inline struct mt76u_buf
311
-*mt76u_get_next_rx_entry(struct mt76_queue *q)
492
+static struct urb *
493
+mt76u_get_next_rx_entry(struct mt76_queue *q)
312494 {
313
-	struct mt76u_buf *buf = NULL;
495
+	struct urb *urb = NULL;
314496 	unsigned long flags;
315497 
316498 	spin_lock_irqsave(&q->lock, flags);
317499 	if (q->queued > 0) {
318
-		buf = &q->entry[q->head].ubuf;
319
-		q->head = (q->head + 1) % q->ndesc;
500
+		urb = q->entry[q->tail].urb;
501
+		q->tail = (q->tail + 1) % q->ndesc;
320502 		q->queued--;
321503 	}
322504 	spin_unlock_irqrestore(&q->lock, flags);
323505 
324
-	return buf;
506
+	return urb;
325507 }
326508 
327
-static int mt76u_get_rx_entry_len(u8 *data, u32 data_len)
509
+static int
510
+mt76u_get_rx_entry_len(struct mt76_dev *dev, u8 *data,
511
+		       u32 data_len)
328512 {
329513 	u16 dma_len, min_len;
330514 
331515 	dma_len = get_unaligned_le16(data);
332
-	min_len = MT_DMA_HDR_LEN + MT_RX_RXWI_LEN +
333
-		  MT_FCE_INFO_LEN;
516
+	if (dev->drv->drv_flags & MT_DRV_RX_DMA_HDR)
517
+		return dma_len;
334518 
335
-	if (data_len < min_len || WARN_ON(!dma_len) ||
336
-	    WARN_ON(dma_len + MT_DMA_HDR_LEN > data_len) ||
337
-	    WARN_ON(dma_len & 0x3))
519
+	min_len = MT_DMA_HDR_LEN + MT_RX_RXWI_LEN + MT_FCE_INFO_LEN;
520
+	if (data_len < min_len || !dma_len ||
521
+	    dma_len + MT_DMA_HDR_LEN > data_len ||
522
+	    (dma_len & 0x3))
338523 		return -EINVAL;
339524 	return dma_len;
340525 }
341526 
342
-static int
343
-mt76u_process_rx_entry(struct mt76_dev *dev, struct urb *urb)
527
+static struct sk_buff *
528
+mt76u_build_rx_skb(struct mt76_dev *dev, void *data,
529
+		   int len, int buf_size)
344530 {
345
-	struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN];
346
-	u8 *data = sg_virt(&urb->sg[0]);
347
-	int data_len, len, nsgs = 1;
531
+	int head_room, drv_flags = dev->drv->drv_flags;
348532 	struct sk_buff *skb;
349533 
350
-	if (!test_bit(MT76_STATE_INITIALIZED, &dev->state))
534
+	head_room = drv_flags & MT_DRV_RX_DMA_HDR ? 0 : MT_DMA_HDR_LEN;
535
+	if (SKB_WITH_OVERHEAD(buf_size) < head_room + len) {
536
+		struct page *page;
537
+
538
+		/* slow path, not enough space for data and
539
+		 * skb_shared_info
540
+		 */
541
+		skb = alloc_skb(MT_SKB_HEAD_LEN, GFP_ATOMIC);
542
+		if (!skb)
543
+			return NULL;
544
+
545
+		skb_put_data(skb, data + head_room, MT_SKB_HEAD_LEN);
546
+		data += head_room + MT_SKB_HEAD_LEN;
547
+		page = virt_to_head_page(data);
548
+		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
549
+				page, data - page_address(page),
550
+				len - MT_SKB_HEAD_LEN, buf_size);
551
+
552
+		return skb;
553
+	}
554
+
555
+	/* fast path */
556
+	skb = build_skb(data, buf_size);
557
+	if (!skb)
558
+		return NULL;
559
+
560
+	skb_reserve(skb, head_room);
561
+	__skb_put(skb, len);
562
+
563
+	return skb;
564
+}
565
+
566
+static int
567
+mt76u_process_rx_entry(struct mt76_dev *dev, struct urb *urb,
568
+		       int buf_size)
569
+{
570
+	u8 *data = urb->num_sgs ? sg_virt(&urb->sg[0]) : urb->transfer_buffer;
571
+	int data_len = urb->num_sgs ? urb->sg[0].length : urb->actual_length;
572
+	int len, nsgs = 1, head_room, drv_flags = dev->drv->drv_flags;
573
+	struct sk_buff *skb;
574
+
575
+	if (!test_bit(MT76_STATE_INITIALIZED, &dev->phy.state))
351576 		return 0;
352577 
353
-	len = mt76u_get_rx_entry_len(data, urb->actual_length);
578
+	len = mt76u_get_rx_entry_len(dev, data, urb->actual_length);
354579 	if (len < 0)
355580 		return 0;
356581 
357
-	skb = build_skb(data, q->buf_size);
582
+	head_room = drv_flags & MT_DRV_RX_DMA_HDR ? 0 : MT_DMA_HDR_LEN;
583
+	data_len = min_t(int, len, data_len - head_room);
584
+	skb = mt76u_build_rx_skb(dev, data, data_len, buf_size);
358585 	if (!skb)
359586 		return 0;
360587 
361
-	data_len = min_t(int, len, urb->sg[0].length - MT_DMA_HDR_LEN);
362
-	skb_reserve(skb, MT_DMA_HDR_LEN);
363
-	if (skb->tail + data_len > skb->end) {
364
-		dev_kfree_skb(skb);
365
-		return 1;
366
-	}
367
-
368
-	__skb_put(skb, data_len);
369588 	len -= data_len;
370
-
371
-	while (len > 0) {
589
+	while (len > 0 && nsgs < urb->num_sgs) {
372590 		data_len = min_t(int, len, urb->sg[nsgs].length);
373591 		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
374592 				sg_page(&urb->sg[nsgs]),
375
-				urb->sg[nsgs].offset,
376
-				data_len, q->buf_size);
593
+				urb->sg[nsgs].offset, data_len,
594
+				buf_size);
377595 		len -= data_len;
378596 		nsgs++;
379597 	}
....@@ -384,9 +602,11 @@
384602 
385603 static void mt76u_complete_rx(struct urb *urb)
386604 {
387
-	struct mt76_dev *dev = urb->context;
388
-	struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN];
605
+	struct mt76_dev *dev = dev_get_drvdata(&urb->dev->dev);
606
+	struct mt76_queue *q = urb->context;
389607 	unsigned long flags;
608
+
609
+	trace_rx_urb(dev, urb);
390610 
391611 	switch (urb->status) {
392612 	case -ECONNRESET:
....@@ -394,65 +614,82 @@
394614 	case -ENOENT:
395615 		return;
396616 	default:
397
-		dev_err(dev->dev, "rx urb failed: %d\n", urb->status);
398
-		/* fall through */
617
+		dev_err_ratelimited(dev->dev, "rx urb failed: %d\n",
618
+				    urb->status);
619
+		fallthrough;
399620 	case 0:
400621 		break;
401622 	}
402623 
403624 	spin_lock_irqsave(&q->lock, flags);
404
-	if (WARN_ONCE(q->entry[q->tail].ubuf.urb != urb, "rx urb mismatch"))
625
+	if (WARN_ONCE(q->entry[q->head].urb != urb, "rx urb mismatch"))
405626 		goto out;
406627 
407
-	q->tail = (q->tail + 1) % q->ndesc;
628
+	q->head = (q->head + 1) % q->ndesc;
408629 	q->queued++;
409630 	tasklet_schedule(&dev->usb.rx_tasklet);
410631 out:
411632 	spin_unlock_irqrestore(&q->lock, flags);
412633 }
413634 
414
-static void mt76u_rx_tasklet(unsigned long data)
635
+static int
636
+mt76u_submit_rx_buf(struct mt76_dev *dev, enum mt76_rxq_id qid,
637
+		    struct urb *urb)
415638 {
416
-	struct mt76_dev *dev = (struct mt76_dev *)data;
417
-	struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN];
418
-	int err, nsgs, buf_len = q->buf_size;
419
-	struct mt76u_buf *buf;
639
+	int ep = qid == MT_RXQ_MAIN ? MT_EP_IN_PKT_RX : MT_EP_IN_CMD_RESP;
420640 
421
-	rcu_read_lock();
641
+	mt76u_fill_bulk_urb(dev, USB_DIR_IN, ep, urb,
642
+			    mt76u_complete_rx, &dev->q_rx[qid]);
643
+	trace_submit_urb(dev, urb);
644
+
645
+	return usb_submit_urb(urb, GFP_ATOMIC);
646
+}
647
+
648
+static void
649
+mt76u_process_rx_queue(struct mt76_dev *dev, struct mt76_queue *q)
650
+{
651
+	int qid = q - &dev->q_rx[MT_RXQ_MAIN];
652
+	struct urb *urb;
653
+	int err, count;
422654 
423655 	while (true) {
424
-		buf = mt76u_get_next_rx_entry(q);
425
-		if (!buf)
656
+		urb = mt76u_get_next_rx_entry(q);
657
+		if (!urb)
426658 			break;
427659 
428
-		nsgs = mt76u_process_rx_entry(dev, buf->urb);
429
-		if (nsgs > 0) {
430
-			err = mt76u_fill_rx_sg(dev, buf, nsgs,
431
-					       buf_len,
432
-					       SKB_WITH_OVERHEAD(buf_len));
660
+		count = mt76u_process_rx_entry(dev, urb, q->buf_size);
661
+		if (count > 0) {
662
+			err = mt76u_refill_rx(dev, q, urb, count, GFP_ATOMIC);
433663 			if (err < 0)
434664 				break;
435665 		}
436
-		mt76u_submit_buf(dev, USB_DIR_IN, MT_EP_IN_PKT_RX,
437
-				 buf, GFP_ATOMIC,
438
-				 mt76u_complete_rx, dev);
666
+		mt76u_submit_rx_buf(dev, qid, urb);
439667 	}
440
-	mt76_rx_poll_complete(dev, MT_RXQ_MAIN, NULL);
668
+	if (qid == MT_RXQ_MAIN)
669
+		mt76_rx_poll_complete(dev, MT_RXQ_MAIN, NULL);
670
+}
441671 
672
+static void mt76u_rx_tasklet(unsigned long data)
673
+{
674
+	struct mt76_dev *dev = (struct mt76_dev *)data;
675
+	int i;
676
+
677
+	rcu_read_lock();
678
+	mt76_for_each_q_rx(dev, i)
679
+		mt76u_process_rx_queue(dev, &dev->q_rx[i]);
442680 	rcu_read_unlock();
443681 }
444682 
445
-int mt76u_submit_rx_buffers(struct mt76_dev *dev)
683
+static int
684
+mt76u_submit_rx_buffers(struct mt76_dev *dev, enum mt76_rxq_id qid)
446685 {
447
-	struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN];
686
+	struct mt76_queue *q = &dev->q_rx[qid];
448687 	unsigned long flags;
449688 	int i, err = 0;
450689 
451690 	spin_lock_irqsave(&q->lock, flags);
452691 	for (i = 0; i < q->ndesc; i++) {
453
-		err = mt76u_submit_buf(dev, USB_DIR_IN, MT_EP_IN_PKT_RX,
454
-				       &q->entry[i].ubuf, GFP_ATOMIC,
455
-				       mt76u_complete_rx, dev);
692
+		err = mt76u_submit_rx_buf(dev, qid, q->entry[i].urb);
456693 		if (err < 0)
457694 			break;
458695 	}
....@@ -462,12 +699,12 @@
462699 
463700 	return err;
464701 }
465
-EXPORT_SYMBOL_GPL(mt76u_submit_rx_buffers);
466702 
467
-static int mt76u_alloc_rx(struct mt76_dev *dev)
703
+static int
704
+mt76u_alloc_rx_queue(struct mt76_dev *dev, enum mt76_rxq_id qid)
468705 {
469
-	struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN];
470
-	int i, err, nsgs;
706
+	struct mt76_queue *q = &dev->q_rx[qid];
707
+	int i, err;
471708 
472709 	spin_lock_init(&q->lock);
473710 	q->entry = devm_kcalloc(dev->dev,
....@@ -476,120 +713,118 @@
476713 	if (!q->entry)
477714 		return -ENOMEM;
478715 
479
-	if (mt76u_check_sg(dev)) {
480
-		q->buf_size = MT_RX_BUF_SIZE;
481
-		nsgs = MT_SG_MAX_SIZE;
482
-	} else {
483
-		q->buf_size = PAGE_SIZE;
484
-		nsgs = 1;
485
-	}
486
-
487716 	q->ndesc = MT_NUM_RX_ENTRIES;
717
+	q->buf_size = PAGE_SIZE;
718
+
488719 	for (i = 0; i < q->ndesc; i++) {
489
-		err = mt76u_buf_alloc(dev, &q->entry[i].ubuf,
490
-				      nsgs, q->buf_size,
491
-				      SKB_WITH_OVERHEAD(q->buf_size),
492
-				      GFP_KERNEL);
720
+		err = mt76u_rx_urb_alloc(dev, q, &q->entry[i]);
493721 		if (err < 0)
494722 			return err;
495723 	}
496724 
497
-	return mt76u_submit_rx_buffers(dev);
725
+	return mt76u_submit_rx_buffers(dev, qid);
726
+}
727
+
728
+int mt76u_alloc_mcu_queue(struct mt76_dev *dev)
729
+{
730
+	return mt76u_alloc_rx_queue(dev, MT_RXQ_MCU);
731
+}
732
+EXPORT_SYMBOL_GPL(mt76u_alloc_mcu_queue);
733
+
734
+static void
735
+mt76u_free_rx_queue(struct mt76_dev *dev, struct mt76_queue *q)
736
+{
737
+	struct page *page;
738
+	int i;
739
+
740
+	for (i = 0; i < q->ndesc; i++)
741
+		mt76u_urb_free(q->entry[i].urb);
742
+
743
+	if (!q->rx_page.va)
744
+		return;
745
+
746
+	page = virt_to_page(q->rx_page.va);
747
+	__page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
748
+	memset(&q->rx_page, 0, sizeof(q->rx_page));
498749 }
499750 
500751 static void mt76u_free_rx(struct mt76_dev *dev)
501752 {
502
-	struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN];
503753 	int i;
504754 
505
-	for (i = 0; i < q->ndesc; i++)
506
-		mt76u_buf_free(&q->entry[i].ubuf);
755
+	mt76_for_each_q_rx(dev, i)
756
+		mt76u_free_rx_queue(dev, &dev->q_rx[i]);
507757 }
508758 
509
-static void mt76u_stop_rx(struct mt76_dev *dev)
759
+void mt76u_stop_rx(struct mt76_dev *dev)
510760 {
511
-	struct mt76_queue *q = &dev->q_rx[MT_RXQ_MAIN];
512761 	int i;
513762 
514
-	for (i = 0; i < q->ndesc; i++)
515
-		usb_kill_urb(q->entry[i].ubuf.urb);
763
+	mt76_for_each_q_rx(dev, i) {
764
+		struct mt76_queue *q = &dev->q_rx[i];
765
+		int j;
766
+
767
+		for (j = 0; j < q->ndesc; j++)
768
+			usb_poison_urb(q->entry[j].urb);
769
+	}
770
+
771
+	tasklet_kill(&dev->usb.rx_tasklet);
516772 }
773
+EXPORT_SYMBOL_GPL(mt76u_stop_rx);
517774 
518
-int mt76u_skb_dma_info(struct sk_buff *skb, int port, u32 flags)
775
+int mt76u_resume_rx(struct mt76_dev *dev)
519776 {
520
-	struct sk_buff *iter, *last = skb;
521
-	u32 info, pad;
777
+	int i;
522778 
523
-	/* Buffer layout:
524
-	 *	|   4B   | xfer len |      pad       |  4B  |
525
-	 *	| TXINFO | pkt/cmd  | zero pad to 4B | zero |
526
-	 *
527
-	 * length field of TXINFO should be set to 'xfer len'.
528
-	 */
529
-	info = FIELD_PREP(MT_TXD_INFO_LEN, round_up(skb->len, 4)) |
530
-	       FIELD_PREP(MT_TXD_INFO_DPORT, port) | flags;
531
-	put_unaligned_le32(info, skb_push(skb, sizeof(info)));
779
+	mt76_for_each_q_rx(dev, i) {
780
+		struct mt76_queue *q = &dev->q_rx[i];
781
+		int err, j;
532782 
533
-	pad = round_up(skb->len, 4) + 4 - skb->len;
534
-	skb_walk_frags(skb, iter) {
535
-		last = iter;
536
-		if (!iter->next) {
537
-			skb->data_len += pad;
538
-			skb->len += pad;
539
-			break;
540
-		}
783
+		for (j = 0; j < q->ndesc; j++)
784
+			usb_unpoison_urb(q->entry[j].urb);
785
+
786
+		err = mt76u_submit_rx_buffers(dev, i);
787
+		if (err < 0)
788
+			return err;
541789 	}
542790 
543
-	if (unlikely(pad)) {
544
-		if (__skb_pad(last, pad, true))
545
-			return -ENOMEM;
546
-		__skb_put(last, pad);
547
-	}
548791 	return 0;
549792 }
550
-EXPORT_SYMBOL_GPL(mt76u_skb_dma_info);
793
+EXPORT_SYMBOL_GPL(mt76u_resume_rx);
551794 
552
-static void mt76u_tx_tasklet(unsigned long data)
795
+static void mt76u_tx_worker(struct mt76_worker *w)
553796 {
554
-	struct mt76_dev *dev = (struct mt76_dev *)data;
555
-	struct mt76u_buf *buf;
797
+	struct mt76_dev *dev = container_of(w, struct mt76_dev, tx_worker);
798
+	struct mt76_queue_entry entry;
556799 	struct mt76_queue *q;
557800 	bool wake;
558801 	int i;
559802 
560803 	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
561
-		q = &dev->q_tx[i];
804
+		q = dev->q_tx[i];
562805 
563
-		spin_lock_bh(&q->lock);
564
-		while (true) {
565
-			buf = &q->entry[q->head].ubuf;
566
-			if (!buf->done || !q->queued)
806
+		while (q->queued > 0) {
807
+			if (!q->entry[q->tail].done)
567808 				break;
568809 
569
-			dev->drv->tx_complete_skb(dev, q,
570
-						  &q->entry[q->head],
571
-						  false);
810
+			entry = q->entry[q->tail];
811
+			q->entry[q->tail].done = false;
572812 
573
-			if (q->entry[q->head].schedule) {
574
-				q->entry[q->head].schedule = false;
575
-				q->swq_queued--;
576
-			}
577
-
578
-			q->head = (q->head + 1) % q->ndesc;
579
-			q->queued--;
813
+			mt76_queue_tx_complete(dev, q, &entry);
580814 		}
581
-		mt76_txq_schedule(dev, q);
582
-		wake = i < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
815
+
816
+		wake = q->stopped && q->queued < q->ndesc - 8;
817
+		if (wake)
818
+			q->stopped = false;
819
+
583820 		if (!q->queued)
584821 			wake_up(&dev->tx_wait);
585822 
586
-		spin_unlock_bh(&q->lock);
823
+		mt76_txq_schedule(&dev->phy, i);
587824 
588
-		if (!test_and_set_bit(MT76_READING_STATS, &dev->state))
589
-			ieee80211_queue_delayed_work(dev->hw,
590
-						     &dev->usb.stat_work,
591
-						     msecs_to_jiffies(10));
592
-
825
+		if (dev->drv->tx_status_data &&
826
+		    !test_and_set_bit(MT76_READING_STATS, &dev->phy.state))
827
+			queue_work(dev->wq, &dev->usb.stat_work);
593828 		if (wake)
594829 			ieee80211_wake_queue(dev->hw, i);
595830 	}
....@@ -602,11 +837,11 @@
602837 	u8 update = 1;
603838 	u16 count = 0;
604839 
605
-	usb = container_of(work, struct mt76_usb, stat_work.work);
840
+	usb = container_of(work, struct mt76_usb, stat_work);
606841 	dev = container_of(usb, struct mt76_dev, usb);
607842 
608843 	while (true) {
609
-		if (test_bit(MT76_REMOVED, &dev->state))
844
+		if (test_bit(MT76_REMOVED, &dev->phy.state))
610845 			break;
611846 
612847 		if (!dev->drv->tx_status_data(dev, &update))
....@@ -614,76 +849,73 @@
614849 		count++;
615850 	}
616851 
617
-	if (count && test_bit(MT76_STATE_RUNNING, &dev->state))
618
-		ieee80211_queue_delayed_work(dev->hw, &usb->stat_work,
619
-					     msecs_to_jiffies(10));
852
+	if (count && test_bit(MT76_STATE_RUNNING, &dev->phy.state))
853
+		queue_work(dev->wq, &usb->stat_work);
620854 	else
621
-		clear_bit(MT76_READING_STATS, &dev->state);
855
+		clear_bit(MT76_READING_STATS, &dev->phy.state);
622856 }
623857 
624858 static void mt76u_complete_tx(struct urb *urb)
625859 {
626
-	struct mt76u_buf *buf = urb->context;
627
-	struct mt76_dev *dev = buf->dev;
860
+	struct mt76_dev *dev = dev_get_drvdata(&urb->dev->dev);
861
+	struct mt76_queue_entry *e = urb->context;
628862 
629863 	if (mt76u_urb_error(urb))
630864 		dev_err(dev->dev, "tx urb failed: %d\n", urb->status);
631
-	buf->done = true;
865
+	e->done = true;
632866 
633
-	tasklet_schedule(&dev->usb.tx_tasklet);
867
+	mt76_worker_schedule(&dev->tx_worker);
634868 }
635869 
636870 static int
637
-mt76u_tx_build_sg(struct sk_buff *skb, struct urb *urb)
871
+mt76u_tx_setup_buffers(struct mt76_dev *dev, struct sk_buff *skb,
872
+		       struct urb *urb)
638873 {
639
-	int nsgs = 1 + skb_shinfo(skb)->nr_frags;
640
-	struct sk_buff *iter;
874
+	urb->transfer_buffer_length = skb->len;
641875 
642
-	skb_walk_frags(skb, iter)
643
-		nsgs += 1 + skb_shinfo(iter)->nr_frags;
876
+	if (!dev->usb.sg_en) {
877
+		urb->transfer_buffer = skb->data;
878
+		return 0;
879
+	}
644880 
645
-	memset(urb->sg, 0, sizeof(*urb->sg) * MT_SG_MAX_SIZE);
881
+	sg_init_table(urb->sg, MT_TX_SG_MAX_SIZE);
882
+	urb->num_sgs = skb_to_sgvec(skb, urb->sg, 0, skb->len);
883
+	if (!urb->num_sgs)
884
+		return -ENOMEM;
646885 
647
-	nsgs = min_t(int, MT_SG_MAX_SIZE, nsgs);
648
-	sg_init_marker(urb->sg, nsgs);
649
-	urb->num_sgs = nsgs;
650
-
651
-	return skb_to_sgvec_nomark(skb, urb->sg, 0, skb->len);
886
+	return urb->num_sgs;
652887 }
653888 
654889 static int
655
-mt76u_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
890
+mt76u_tx_queue_skb(struct mt76_dev *dev, enum mt76_txq_id qid,
656891 		   struct sk_buff *skb, struct mt76_wcid *wcid,
657892 		   struct ieee80211_sta *sta)
658893 {
659
-	struct usb_interface *intf = to_usb_interface(dev->dev);
660
-	struct usb_device *udev = interface_to_usbdev(intf);
661
-	u8 ep = q2ep(q->hw_idx);
662
-	struct mt76u_buf *buf;
663
-	u16 idx = q->tail;
664
-	unsigned int pipe;
894
+	struct mt76_queue *q = dev->q_tx[qid];
895
+	struct mt76_tx_info tx_info = {
896
+		.skb = skb,
897
+	};
898
+	u16 idx = q->head;
665899 	int err;
666900 
667901 	if (q->queued == q->ndesc)
668902 		return -ENOSPC;
669903 
670
-	err = dev->drv->tx_prepare_skb(dev, NULL, skb, q, wcid, sta, NULL);
904
+	skb->prev = skb->next = NULL;
905
+	err = dev->drv->tx_prepare_skb(dev, NULL, qid, wcid, sta, &tx_info);
671906 	if (err < 0)
672907 		return err;
673908 
674
-	buf = &q->entry[idx].ubuf;
675
-	buf->done = false;
676
-
677
-	err = mt76u_tx_build_sg(skb, buf->urb);
909
+	err = mt76u_tx_setup_buffers(dev, tx_info.skb, q->entry[idx].urb);
678910 	if (err < 0)
679911 		return err;
680912 
681
-	pipe = usb_sndbulkpipe(udev, dev->usb.out_ep[ep]);
682
-	usb_fill_bulk_urb(buf->urb, udev, pipe, NULL, skb->len,
683
-			  mt76u_complete_tx, buf);
913
+	mt76u_fill_bulk_urb(dev, USB_DIR_OUT, q2ep(q->hw_idx),
914
+			    q->entry[idx].urb, mt76u_complete_tx,
915
+			    &q->entry[idx]);
684916 
685
-	q->tail = (q->tail + 1) % q->ndesc;
686
-	q->entry[idx].skb = skb;
917
+	q->head = (q->head + 1) % q->ndesc;
918
+	q->entry[idx].skb = tx_info.skb;
687919 	q->queued++;
688920 
689921 	return idx;
....@@ -691,15 +923,17 @@
691923 
692924 static void mt76u_tx_kick(struct mt76_dev *dev, struct mt76_queue *q)
693925 {
694
-	struct mt76u_buf *buf;
926
+	struct urb *urb;
695927 	int err;
696928 
697
-	while (q->first != q->tail) {
698
-		buf = &q->entry[q->first].ubuf;
699
-		err = usb_submit_urb(buf->urb, GFP_ATOMIC);
929
+	while (q->first != q->head) {
930
+		urb = q->entry[q->first].urb;
931
+
932
+		trace_submit_urb(dev, urb);
933
+		err = usb_submit_urb(urb, GFP_ATOMIC);
700934 		if (err < 0) {
701935 			if (err == -ENODEV)
702
-				set_bit(MT76_REMOVED, &dev->state);
936
+				set_bit(MT76_REMOVED, &dev->phy.state);
703937 			else
704938 				dev_err(dev->dev, "tx urb submit failed:%d\n",
705939 					err);
....@@ -709,19 +943,44 @@
709943 	}
710944 }
711945 
946
+static u8 mt76u_ac_to_hwq(struct mt76_dev *dev, u8 ac)
947
+{
948
+	if (mt76_chip(dev) == 0x7663) {
949
+		static const u8 lmac_queue_map[] = {
950
+			/* ac to lmac mapping */
951
+			[IEEE80211_AC_BK] = 0,
952
+			[IEEE80211_AC_BE] = 1,
953
+			[IEEE80211_AC_VI] = 2,
954
+			[IEEE80211_AC_VO] = 4,
955
+		};
956
+
957
+		if (WARN_ON(ac >= ARRAY_SIZE(lmac_queue_map)))
958
+			return 1; /* BE */
959
+
960
+		return lmac_queue_map[ac];
961
+	}
962
+
963
+	return mt76_ac_to_hwq(ac);
964
+}
965
+
712966 static int mt76u_alloc_tx(struct mt76_dev *dev)
713967 {
714
-	struct mt76u_buf *buf;
715968 	struct mt76_queue *q;
716
-	size_t size;
717
-	int i, j;
969
+	int i, j, err;
718970 
719
-	size = MT_SG_MAX_SIZE * sizeof(struct scatterlist);
720
-	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
721
-		q = &dev->q_tx[i];
971
+	for (i = 0; i <= MT_TXQ_PSD; i++) {
972
+		if (i >= IEEE80211_NUM_ACS) {
973
+			dev->q_tx[i] = dev->q_tx[0];
974
+			continue;
975
+		}
976
+
977
+		q = devm_kzalloc(dev->dev, sizeof(*q), GFP_KERNEL);
978
+		if (!q)
979
+			return -ENOMEM;
980
+
722981 		spin_lock_init(&q->lock);
723
-		INIT_LIST_HEAD(&q->swq);
724
-		q->hw_idx = q2hwq(i);
982
+		q->hw_idx = mt76u_ac_to_hwq(dev, i);
983
+		dev->q_tx[i] = q;
725984 
726985 		q->entry = devm_kcalloc(dev->dev,
727986 					MT_NUM_TX_ENTRIES, sizeof(*q->entry),
....@@ -731,16 +990,10 @@
731990 
732991 		q->ndesc = MT_NUM_TX_ENTRIES;
733992 		for (j = 0; j < q->ndesc; j++) {
734
-			buf = &q->entry[j].ubuf;
735
-			buf->dev = dev;
736
-
737
-			buf->urb = usb_alloc_urb(0, GFP_KERNEL);
738
-			if (!buf->urb)
739
-				return -ENOMEM;
740
-
741
-			buf->urb->sg = devm_kzalloc(dev->dev, size, GFP_KERNEL);
742
-			if (!buf->urb->sg)
743
-				return -ENOMEM;
993
+			err = mt76u_urb_alloc(dev, &q->entry[j],
994
+					      MT_TX_SG_MAX_SIZE);
995
+			if (err < 0)
996
+				return err;
744997 		}
745998 	}
746999 	return 0;
....@@ -748,48 +1001,74 @@
7481001 
7491002 static void mt76u_free_tx(struct mt76_dev *dev)
7501003 {
751
-	struct mt76_queue *q;
752
-	int i, j;
1004
+	int i;
7531005 
7541006 	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
755
-		q = &dev->q_tx[i];
1007
+		struct mt76_queue *q;
1008
+		int j;
1009
+
1010
+		q = dev->q_tx[i];
1011
+		if (!q)
1012
+			continue;
1013
+
7561014 		for (j = 0; j < q->ndesc; j++)
757
-			usb_free_urb(q->entry[j].ubuf.urb);
1015
+			usb_free_urb(q->entry[j].urb);
7581016 	}
7591017 }
7601018 
761
-static void mt76u_stop_tx(struct mt76_dev *dev)
1019
+void mt76u_stop_tx(struct mt76_dev *dev)
7621020 {
763
-	struct mt76_queue *q;
764
-	int i, j;
1021
+	int ret;
7651022 
766
-	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
767
-		q = &dev->q_tx[i];
768
-		for (j = 0; j < q->ndesc; j++)
769
-			usb_kill_urb(q->entry[j].ubuf.urb);
1023
+	ret = wait_event_timeout(dev->tx_wait, !mt76_has_tx_pending(&dev->phy),
1024
+				 HZ / 5);
1025
+	if (!ret) {
1026
+		struct mt76_queue_entry entry;
1027
+		struct mt76_queue *q;
1028
+		int i, j;
1029
+
1030
+		dev_err(dev->dev, "timed out waiting for pending tx\n");
1031
+
1032
+		for (i = 0; i < IEEE80211_NUM_ACS; i++) {
1033
+			q = dev->q_tx[i];
1034
+			if (!q)
1035
+				continue;
1036
+
1037
+			for (j = 0; j < q->ndesc; j++)
1038
+				usb_kill_urb(q->entry[j].urb);
1039
+		}
1040
+
1041
+		mt76_worker_disable(&dev->tx_worker);
1042
+
1043
+		/* On device removal we maight queue skb's, but mt76u_tx_kick()
1044
+		 * will fail to submit urb, cleanup those skb's manually.
1045
+		 */
1046
+		for (i = 0; i < IEEE80211_NUM_ACS; i++) {
1047
+			q = dev->q_tx[i];
1048
+			if (!q)
1049
+				continue;
1050
+
1051
+			while (q->queued > 0) {
1052
+				entry = q->entry[q->tail];
1053
+				q->entry[q->tail].done = false;
1054
+				mt76_queue_tx_complete(dev, q, &entry);
1055
+			}
1056
+		}
1057
+
1058
+		mt76_worker_enable(&dev->tx_worker);
7701059 	}
771
-}
7721060 
773
-void mt76u_stop_queues(struct mt76_dev *dev)
774
-{
775
-	tasklet_disable(&dev->usb.rx_tasklet);
776
-	tasklet_disable(&dev->usb.tx_tasklet);
1061
+	cancel_work_sync(&dev->usb.stat_work);
1062
+	clear_bit(MT76_READING_STATS, &dev->phy.state);
7771063 
778
-	mt76u_stop_rx(dev);
779
-	mt76u_stop_tx(dev);
1064
+	mt76_tx_status_check(dev, NULL, true);
7801065 }
781
-EXPORT_SYMBOL_GPL(mt76u_stop_queues);
782
-
783
-void mt76u_stop_stat_wk(struct mt76_dev *dev)
784
-{
785
-	cancel_delayed_work_sync(&dev->usb.stat_work);
786
-	clear_bit(MT76_READING_STATS, &dev->state);
787
-}
788
-EXPORT_SYMBOL_GPL(mt76u_stop_stat_wk);
1066
+EXPORT_SYMBOL_GPL(mt76u_stop_tx);
7891067 
7901068 void mt76u_queues_deinit(struct mt76_dev *dev)
7911069 {
792
-	mt76u_stop_queues(dev);
1070
+	mt76u_stop_rx(dev);
1071
+	mt76u_stop_tx(dev);
7931072 
7941073 	mt76u_free_rx(dev);
7951074 	mt76u_free_tx(dev);
....@@ -800,7 +1079,7 @@
8001079 {
8011080 	int err;
8021081 
803
-	err = mt76u_alloc_rx(dev);
1082
+	err = mt76u_alloc_rx_queue(dev, MT_RXQ_MAIN);
8041083 	if (err < 0)
8051084 		return err;
8061085 
....@@ -814,29 +1093,53 @@
8141093 };
8151094 
8161095 int mt76u_init(struct mt76_dev *dev,
817
-	       struct usb_interface *intf)
1096
+	       struct usb_interface *intf, bool ext)
8181097 {
819
-	static const struct mt76_bus_ops mt76u_ops = {
820
-		.rr = mt76u_rr,
821
-		.wr = mt76u_wr,
822
-		.rmw = mt76u_rmw,
823
-		.copy = mt76u_copy,
1098
+	static struct mt76_bus_ops mt76u_ops = {
1099
+		.read_copy = mt76u_read_copy_ext,
1100
+		.wr_rp = mt76u_wr_rp,
1101
+		.rd_rp = mt76u_rd_rp,
1102
+		.type = MT76_BUS_USB,
8241103 	};
1104
+	struct usb_device *udev = interface_to_usbdev(intf);
8251105 	struct mt76_usb *usb = &dev->usb;
1106
+	int err = -ENOMEM;
8261107 
1108
+	mt76u_ops.rr = ext ? mt76u_rr_ext : mt76u_rr;
1109
+	mt76u_ops.wr = ext ? mt76u_wr_ext : mt76u_wr;
1110
+	mt76u_ops.rmw = ext ? mt76u_rmw_ext : mt76u_rmw;
1111
+	mt76u_ops.write_copy = ext ? mt76u_copy_ext : mt76u_copy;
1112
+
1113
+	dev->tx_worker.fn = mt76u_tx_worker;
8271114 	tasklet_init(&usb->rx_tasklet, mt76u_rx_tasklet, (unsigned long)dev);
828
-	tasklet_init(&usb->tx_tasklet, mt76u_tx_tasklet, (unsigned long)dev);
829
-	INIT_DELAYED_WORK(&usb->stat_work, mt76u_tx_status_data);
830
-	skb_queue_head_init(&dev->rx_skb[MT_RXQ_MAIN]);
1115
+	INIT_WORK(&usb->stat_work, mt76u_tx_status_data);
8311116 
832
-	init_completion(&usb->mcu.cmpl);
833
-	mutex_init(&usb->mcu.mutex);
1117
+	usb->data_len = usb_maxpacket(udev, usb_sndctrlpipe(udev, 0), 1);
1118
+	if (usb->data_len < 32)
1119
+		usb->data_len = 32;
1120
+
1121
+	usb->data = devm_kmalloc(dev->dev, usb->data_len, GFP_KERNEL);
1122
+	if (!usb->data)
1123
+		goto error;
8341124 
8351125 	mutex_init(&usb->usb_ctrl_mtx);
8361126 	dev->bus = &mt76u_ops;
8371127 	dev->queue_ops = &usb_queue_ops;
8381128 
839
-	return mt76u_set_endpoints(intf, usb);
1129
+	dev_set_drvdata(&udev->dev, dev);
1130
+
1131
+	usb->sg_en = mt76u_check_sg(dev);
1132
+
1133
+	err = mt76u_set_endpoints(intf, usb);
1134
+	if (err < 0)
1135
+		goto error;
1136
+
1137
+	return 0;
1138
+
1139
+error:
1140
+	destroy_workqueue(dev->wq);
1141
+
1142
+	return err;
8401143 }
8411144 EXPORT_SYMBOL_GPL(mt76u_init);
8421145