forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/net/ethernet/intel/ice/ice_controlq.c
....@@ -3,6 +3,28 @@
33 
44 #include "ice_common.h"
55 
6
+#define ICE_CQ_INIT_REGS(qinfo, prefix)				\
7
+do {								\
8
+	(qinfo)->sq.head = prefix##_ATQH;			\
9
+	(qinfo)->sq.tail = prefix##_ATQT;			\
10
+	(qinfo)->sq.len = prefix##_ATQLEN;			\
11
+	(qinfo)->sq.bah = prefix##_ATQBAH;			\
12
+	(qinfo)->sq.bal = prefix##_ATQBAL;			\
13
+	(qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M;	\
14
+	(qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M;	\
15
+	(qinfo)->sq.len_crit_mask = prefix##_ATQLEN_ATQCRIT_M;	\
16
+	(qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M;		\
17
+	(qinfo)->rq.head = prefix##_ARQH;			\
18
+	(qinfo)->rq.tail = prefix##_ARQT;			\
19
+	(qinfo)->rq.len = prefix##_ARQLEN;			\
20
+	(qinfo)->rq.bah = prefix##_ARQBAH;			\
21
+	(qinfo)->rq.bal = prefix##_ARQBAL;			\
22
+	(qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M;	\
23
+	(qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M;	\
24
+	(qinfo)->rq.len_crit_mask = prefix##_ARQLEN_ARQCRIT_M;	\
25
+	(qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M;		\
26
+} while (0)
27
+
628 /**
729  * ice_adminq_init_regs - Initialize AdminQ registers
830  * @hw: pointer to the hardware structure
....@@ -13,28 +35,25 @@
1335 {
1436 	struct ice_ctl_q_info *cq = &hw->adminq;
1537 
16
-	cq->sq.head = PF_FW_ATQH;
17
-	cq->sq.tail = PF_FW_ATQT;
18
-	cq->sq.len = PF_FW_ATQLEN;
19
-	cq->sq.bah = PF_FW_ATQBAH;
20
-	cq->sq.bal = PF_FW_ATQBAL;
21
-	cq->sq.len_mask = PF_FW_ATQLEN_ATQLEN_M;
22
-	cq->sq.len_ena_mask = PF_FW_ATQLEN_ATQENABLE_M;
23
-	cq->sq.head_mask = PF_FW_ATQH_ATQH_M;
38
+	ICE_CQ_INIT_REGS(cq, PF_FW);
39
+}
2440 
25
-	cq->rq.head = PF_FW_ARQH;
26
-	cq->rq.tail = PF_FW_ARQT;
27
-	cq->rq.len = PF_FW_ARQLEN;
28
-	cq->rq.bah = PF_FW_ARQBAH;
29
-	cq->rq.bal = PF_FW_ARQBAL;
30
-	cq->rq.len_mask = PF_FW_ARQLEN_ARQLEN_M;
31
-	cq->rq.len_ena_mask = PF_FW_ARQLEN_ARQENABLE_M;
32
-	cq->rq.head_mask = PF_FW_ARQH_ARQH_M;
41
+/**
42
+ * ice_mailbox_init_regs - Initialize Mailbox registers
43
+ * @hw: pointer to the hardware structure
44
+ *
45
+ * This assumes the alloc_sq and alloc_rq functions have already been called
46
+ */
47
+static void ice_mailbox_init_regs(struct ice_hw *hw)
48
+{
49
+	struct ice_ctl_q_info *cq = &hw->mailboxq;
50
+
51
+	ICE_CQ_INIT_REGS(cq, PF_MBX);
3352 }
3453 
3554 /**
3655  * ice_check_sq_alive
37
- * @hw: pointer to the hw struct
56
+ * @hw: pointer to the HW struct
3857  * @cq: pointer to the specific Control queue
3958  *
4059  * Returns true if Queue is enabled else false.
....@@ -101,37 +120,20 @@
101120 }
102121 
103122 /**
104
- * ice_free_ctrlq_sq_ring - Free Control Transmit Queue (ATQ) rings
123
+ * ice_free_cq_ring - Free control queue ring
105124  * @hw: pointer to the hardware structure
106
- * @cq: pointer to the specific Control queue
125
+ * @ring: pointer to the specific control queue ring
107126  *
108
- * This assumes the posted send buffers have already been cleaned
127
+ * This assumes the posted buffers have already been cleaned
109128  * and de-allocated
110129  */
111
-static void ice_free_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
130
+static void ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring)
112131 {
113
-	dmam_free_coherent(ice_hw_to_dev(hw), cq->sq.desc_buf.size,
114
-			   cq->sq.desc_buf.va, cq->sq.desc_buf.pa);
115
-	cq->sq.desc_buf.va = NULL;
116
-	cq->sq.desc_buf.pa = 0;
117
-	cq->sq.desc_buf.size = 0;
118
-}
119
-
120
-/**
121
- * ice_free_ctrlq_rq_ring - Free Control Receive Queue (ARQ) rings
122
- * @hw: pointer to the hardware structure
123
- * @cq: pointer to the specific Control queue
124
- *
125
- * This assumes the posted receive buffers have already been cleaned
126
- * and de-allocated
127
- */
128
-static void ice_free_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)
129
-{
130
-	dmam_free_coherent(ice_hw_to_dev(hw), cq->rq.desc_buf.size,
131
-			   cq->rq.desc_buf.va, cq->rq.desc_buf.pa);
132
-	cq->rq.desc_buf.va = NULL;
133
-	cq->rq.desc_buf.pa = 0;
134
-	cq->rq.desc_buf.size = 0;
132
+	dmam_free_coherent(ice_hw_to_dev(hw), ring->desc_buf.size,
133
+			   ring->desc_buf.va, ring->desc_buf.pa);
134
+	ring->desc_buf.va = NULL;
135
+	ring->desc_buf.pa = 0;
136
+	ring->desc_buf.size = 0;
135137 }
136138 
137139 /**
....@@ -199,7 +201,9 @@
199201 		cq->rq.r.rq_bi[i].pa = 0;
200202 		cq->rq.r.rq_bi[i].size = 0;
201203 	}
204
+	cq->rq.r.rq_bi = NULL;
202205 	devm_kfree(ice_hw_to_dev(hw), cq->rq.dma_head);
206
+	cq->rq.dma_head = NULL;
203207 
204208 	return ICE_ERR_NO_MEMORY;
205209 }
....@@ -245,59 +249,30 @@
245249 		cq->sq.r.sq_bi[i].pa = 0;
246250 		cq->sq.r.sq_bi[i].size = 0;
247251 	}
252
+	cq->sq.r.sq_bi = NULL;
248253 	devm_kfree(ice_hw_to_dev(hw), cq->sq.dma_head);
254
+	cq->sq.dma_head = NULL;
249255 
250256 	return ICE_ERR_NO_MEMORY;
251257 }
252258 
253
-/**
254
- * ice_free_rq_bufs - Free ARQ buffer info elements
255
- * @hw: pointer to the hardware structure
256
- * @cq: pointer to the specific Control queue
257
- */
258
-static void ice_free_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
259
+static enum ice_status
260
+ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring, u16 num_entries)
259261 {
260
-	int i;
262
+	/* Clear Head and Tail */
263
+	wr32(hw, ring->head, 0);
264
+	wr32(hw, ring->tail, 0);
261265 
262
-	/* free descriptors */
263
-	for (i = 0; i < cq->num_rq_entries; i++) {
264
-		dmam_free_coherent(ice_hw_to_dev(hw), cq->rq.r.rq_bi[i].size,
265
-				   cq->rq.r.rq_bi[i].va, cq->rq.r.rq_bi[i].pa);
266
-		cq->rq.r.rq_bi[i].va = NULL;
267
-		cq->rq.r.rq_bi[i].pa = 0;
268
-		cq->rq.r.rq_bi[i].size = 0;
269
-	}
266
+	/* set starting point */
267
+	wr32(hw, ring->len, (num_entries | ring->len_ena_mask));
268
+	wr32(hw, ring->bal, lower_32_bits(ring->desc_buf.pa));
269
+	wr32(hw, ring->bah, upper_32_bits(ring->desc_buf.pa));
270270 
271
-	/* free the dma header */
272
-	devm_kfree(ice_hw_to_dev(hw), cq->rq.dma_head);
273
-}
271
+	/* Check one register to verify that config was applied */
272
+	if (rd32(hw, ring->bal) != lower_32_bits(ring->desc_buf.pa))
273
+		return ICE_ERR_AQ_ERROR;
274274 
275
-/**
276
- * ice_free_sq_bufs - Free ATQ buffer info elements
277
- * @hw: pointer to the hardware structure
278
- * @cq: pointer to the specific Control queue
279
- */
280
-static void ice_free_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
281
-{
282
-	int i;
283
-
284
-	/* only unmap if the address is non-NULL */
285
-	for (i = 0; i < cq->num_sq_entries; i++)
286
-		if (cq->sq.r.sq_bi[i].pa) {
287
-			dmam_free_coherent(ice_hw_to_dev(hw),
288
-					   cq->sq.r.sq_bi[i].size,
289
-					   cq->sq.r.sq_bi[i].va,
290
-					   cq->sq.r.sq_bi[i].pa);
291
-			cq->sq.r.sq_bi[i].va = NULL;
292
-			cq->sq.r.sq_bi[i].pa = 0;
293
-			cq->sq.r.sq_bi[i].size = 0;
294
-		}
295
-
296
-	/* free the buffer info list */
297
-	devm_kfree(ice_hw_to_dev(hw), cq->sq.cmd_buf);
298
-
299
-	/* free the dma header */
300
-	devm_kfree(ice_hw_to_dev(hw), cq->sq.dma_head);
275
+	return 0;
301276 }
302277 
303278 /**
....@@ -310,23 +285,7 @@
310285 static enum ice_status
311286 ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
312287 {
313
-	u32 reg = 0;
314
-
315
-	/* Clear Head and Tail */
316
-	wr32(hw, cq->sq.head, 0);
317
-	wr32(hw, cq->sq.tail, 0);
318
-
319
-	/* set starting point */
320
-	wr32(hw, cq->sq.len, (cq->num_sq_entries | cq->sq.len_ena_mask));
321
-	wr32(hw, cq->sq.bal, lower_32_bits(cq->sq.desc_buf.pa));
322
-	wr32(hw, cq->sq.bah, upper_32_bits(cq->sq.desc_buf.pa));
323
-
324
-	/* Check one register to verify that config was applied */
325
-	reg = rd32(hw, cq->sq.bal);
326
-	if (reg != lower_32_bits(cq->sq.desc_buf.pa))
327
-		return ICE_ERR_AQ_ERROR;
328
-
329
-	return 0;
288
+	return ice_cfg_cq_regs(hw, &cq->sq, cq->num_sq_entries);
330289 }
331290 
332291 /**
....@@ -334,32 +293,46 @@
334293  * @hw: pointer to the hardware structure
335294  * @cq: pointer to the specific Control queue
336295  *
337
- * Configure base address and length registers for the receive (event q)
296
+ * Configure base address and length registers for the receive (event queue)
338297  */
339298 static enum ice_status
340299 ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)
341300 {
342
-	u32 reg = 0;
301
+	enum ice_status status;
343302 
344
-	/* Clear Head and Tail */
345
-	wr32(hw, cq->rq.head, 0);
346
-	wr32(hw, cq->rq.tail, 0);
347
-
348
-	/* set starting point */
349
-	wr32(hw, cq->rq.len, (cq->num_rq_entries | cq->rq.len_ena_mask));
350
-	wr32(hw, cq->rq.bal, lower_32_bits(cq->rq.desc_buf.pa));
351
-	wr32(hw, cq->rq.bah, upper_32_bits(cq->rq.desc_buf.pa));
303
+	status = ice_cfg_cq_regs(hw, &cq->rq, cq->num_rq_entries);
304
+	if (status)
305
+		return status;
352306 
353307 	/* Update tail in the HW to post pre-allocated buffers */
354308 	wr32(hw, cq->rq.tail, (u32)(cq->num_rq_entries - 1));
355309 
356
-	/* Check one register to verify that config was applied */
357
-	reg = rd32(hw, cq->rq.bal);
358
-	if (reg != lower_32_bits(cq->rq.desc_buf.pa))
359
-		return ICE_ERR_AQ_ERROR;
360
-
361310 	return 0;
362311 }
312
+
313
+#define ICE_FREE_CQ_BUFS(hw, qi, ring)					\
314
+do {									\
315
+	/* free descriptors */						\
316
+	if ((qi)->ring.r.ring##_bi) {					\
317
+		int i;							\
318
+									\
319
+		for (i = 0; i < (qi)->num_##ring##_entries; i++)	\
320
+			if ((qi)->ring.r.ring##_bi[i].pa) {		\
321
+				dmam_free_coherent(ice_hw_to_dev(hw),	\
322
+					(qi)->ring.r.ring##_bi[i].size,	\
323
+					(qi)->ring.r.ring##_bi[i].va,	\
324
+					(qi)->ring.r.ring##_bi[i].pa);	\
325
+					(qi)->ring.r.ring##_bi[i].va = NULL;\
326
+					(qi)->ring.r.ring##_bi[i].pa = 0;\
327
+					(qi)->ring.r.ring##_bi[i].size = 0;\
328
+		}							\
329
+	}								\
330
+	/* free the buffer info list */					\
331
+	if ((qi)->ring.cmd_buf)						\
332
+		devm_kfree(ice_hw_to_dev(hw), (qi)->ring.cmd_buf);	\
333
+	/* free DMA head */						\
334
+	devm_kfree(ice_hw_to_dev(hw), (qi)->ring.dma_head);		\
335
+} while (0)
363336 
364337 /**
365338  * ice_init_sq - main initialization routine for Control ATQ
....@@ -367,7 +340,7 @@
367340  * @cq: pointer to the specific Control queue
368341  *
369342  * This is the main initialization routine for the Control Send Queue
370
- * Prior to calling this function, drivers *MUST* set the following fields
343
+ * Prior to calling this function, the driver *MUST* set the following fields
371344  * in the cq->structure:
372345  *     - cq->num_sq_entries
373346  *     - cq->sq_buf_size
....@@ -414,7 +387,8 @@
414387 	goto init_ctrlq_exit;
415388 
416389 init_ctrlq_free_rings:
417
-	ice_free_ctrlq_sq_ring(hw, cq);
390
+	ICE_FREE_CQ_BUFS(hw, cq, sq);
391
+	ice_free_cq_ring(hw, &cq->sq);
418392 
419393 init_ctrlq_exit:
420394 	return ret_code;
....@@ -426,7 +400,7 @@
426400  * @cq: pointer to the specific Control queue
427401  *
428402  * The main initialization routine for the Admin Receive (Event) Queue.
429
- * Prior to calling this function, drivers *MUST* set the following fields
403
+ * Prior to calling this function, the driver *MUST* set the following fields
430404  * in the cq->structure:
431405  *     - cq->num_rq_entries
432406  *     - cq->rq_buf_size
....@@ -473,7 +447,8 @@
473447 	goto init_ctrlq_exit;
474448 
475449 init_ctrlq_free_rings:
476
-	ice_free_ctrlq_rq_ring(hw, cq);
450
+	ICE_FREE_CQ_BUFS(hw, cq, rq);
451
+	ice_free_cq_ring(hw, &cq->rq);
477452 
478453 init_ctrlq_exit:
479454 	return ret_code;
....@@ -508,8 +483,8 @@
508483 	cq->sq.count = 0;	/* to indicate uninitialized queue */
509484 
510485 	/* free ring buffers and the ring itself */
511
-	ice_free_sq_bufs(hw, cq);
512
-	ice_free_ctrlq_sq_ring(hw, cq);
486
+	ICE_FREE_CQ_BUFS(hw, cq, sq);
487
+	ice_free_cq_ring(hw, &cq->sq);
513488 
514489 shutdown_sq_out:
515490 	mutex_unlock(&cq->sq_lock);
....@@ -576,8 +551,8 @@
576551 	cq->rq.count = 0;
577552 
578553 	/* free ring buffers and the ring itself */
579
-	ice_free_rq_bufs(hw, cq);
580
-	ice_free_ctrlq_rq_ring(hw, cq);
554
+	ICE_FREE_CQ_BUFS(hw, cq, rq);
555
+	ice_free_cq_ring(hw, &cq->rq);
581556 
582557 shutdown_rq_out:
583558 	mutex_unlock(&cq->rq_lock);
....@@ -605,14 +580,8 @@
605580 	return 0;
606581 
607582 init_ctrlq_free_rq:
608
-	if (cq->rq.head) {
609
-		ice_shutdown_rq(hw, cq);
610
-		mutex_destroy(&cq->rq_lock);
611
-	}
612
-	if (cq->sq.head) {
613
-		ice_shutdown_sq(hw, cq);
614
-		mutex_destroy(&cq->sq_lock);
615
-	}
583
+	ice_shutdown_rq(hw, cq);
584
+	ice_shutdown_sq(hw, cq);
616585 	return status;
617586 }
618587 
....@@ -621,13 +590,14 @@
621590  * @hw: pointer to the hardware structure
622591  * @q_type: specific Control queue type
623592  *
624
- * Prior to calling this function, drivers *MUST* set the following fields
593
+ * Prior to calling this function, the driver *MUST* set the following fields
625594  * in the cq->structure:
626595  *     - cq->num_sq_entries
627596  *     - cq->num_rq_entries
628597  *     - cq->rq_buf_size
629598  *     - cq->sq_buf_size
630599  *
600
+ * NOTE: this function does not initialize the controlq locks
631601  */
632602 static enum ice_status ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
633603 {
....@@ -639,6 +609,10 @@
639609 		ice_adminq_init_regs(hw);
640610 		cq = &hw->adminq;
641611 		break;
612
+	case ICE_CTL_Q_MAILBOX:
613
+		ice_mailbox_init_regs(hw);
614
+		cq = &hw->mailboxq;
615
+		break;
642616 	default:
643617 		return ICE_ERR_PARAM;
644618 	}
....@@ -649,8 +623,6 @@
649623 	    !cq->rq_buf_size || !cq->sq_buf_size) {
650624 		return ICE_ERR_CFG;
651625 	}
652
-	mutex_init(&cq->sq_lock);
653
-	mutex_init(&cq->rq_lock);
654626 
655627 	/* setup SQ command write back timeout */
656628 	cq->sq_cmd_timeout = ICE_CTL_Q_SQ_CMD_TIMEOUT;
....@@ -658,7 +630,7 @@
658630 	/* allocate the ATQ */
659631 	ret_code = ice_init_sq(hw, cq);
660632 	if (ret_code)
661
-		goto init_ctrlq_destroy_locks;
633
+		return ret_code;
662634 
663635 	/* allocate the ARQ */
664636 	ret_code = ice_init_rq(hw, cq);
....@@ -670,39 +642,15 @@
670642 
671643 init_ctrlq_free_sq:
672644 	ice_shutdown_sq(hw, cq);
673
-init_ctrlq_destroy_locks:
674
-	mutex_destroy(&cq->sq_lock);
675
-	mutex_destroy(&cq->rq_lock);
676645 	return ret_code;
677
-}
678
-
679
-/**
680
- * ice_init_all_ctrlq - main initialization routine for all control queues
681
- * @hw: pointer to the hardware structure
682
- *
683
- * Prior to calling this function, drivers *MUST* set the following fields
684
- * in the cq->structure for all control queues:
685
- *     - cq->num_sq_entries
686
- *     - cq->num_rq_entries
687
- *     - cq->rq_buf_size
688
- *     - cq->sq_buf_size
689
- */
690
-enum ice_status ice_init_all_ctrlq(struct ice_hw *hw)
691
-{
692
-	enum ice_status ret_code;
693
-
694
-	/* Init FW admin queue */
695
-	ret_code = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN);
696
-	if (ret_code)
697
-		return ret_code;
698
-
699
-	return ice_init_check_adminq(hw);
700646 }
701647 
702648 /**
703649  * ice_shutdown_ctrlq - shutdown routine for any control queue
704650  * @hw: pointer to the hardware structure
705651  * @q_type: specific Control queue type
652
+ *
653
+ * NOTE: this function does not destroy the control queue locks.
706654  */
707655 static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)
708656 {
....@@ -714,28 +662,137 @@
714662 		if (ice_check_sq_alive(hw, cq))
715663 			ice_aq_q_shutdown(hw, true);
716664 		break;
665
+	case ICE_CTL_Q_MAILBOX:
666
+		cq = &hw->mailboxq;
667
+		break;
717668 	default:
718669 		return;
719670 	}
720671 
721
-	if (cq->sq.head) {
722
-		ice_shutdown_sq(hw, cq);
723
-		mutex_destroy(&cq->sq_lock);
724
-	}
725
-	if (cq->rq.head) {
726
-		ice_shutdown_rq(hw, cq);
727
-		mutex_destroy(&cq->rq_lock);
728
-	}
672
+	ice_shutdown_sq(hw, cq);
673
+	ice_shutdown_rq(hw, cq);
729674 }
730675 
731676 /**
732677  * ice_shutdown_all_ctrlq - shutdown routine for all control queues
733678  * @hw: pointer to the hardware structure
679
+ *
680
+ * NOTE: this function does not destroy the control queue locks. The driver
681
+ * may call this at runtime to shutdown and later restart control queues, such
682
+ * as in response to a reset event.
734683  */
735684 void ice_shutdown_all_ctrlq(struct ice_hw *hw)
736685 {
737686 	/* Shutdown FW admin queue */
738687 	ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
688
+	/* Shutdown PF-VF Mailbox */
689
+	ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX);
690
+}
691
+
692
+/**
693
+ * ice_init_all_ctrlq - main initialization routine for all control queues
694
+ * @hw: pointer to the hardware structure
695
+ *
696
+ * Prior to calling this function, the driver MUST* set the following fields
697
+ * in the cq->structure for all control queues:
698
+ *     - cq->num_sq_entries
699
+ *     - cq->num_rq_entries
700
+ *     - cq->rq_buf_size
701
+ *     - cq->sq_buf_size
702
+ *
703
+ * NOTE: this function does not initialize the controlq locks.
704
+ */
705
+enum ice_status ice_init_all_ctrlq(struct ice_hw *hw)
706
+{
707
+	enum ice_status status;
708
+	u32 retry = 0;
709
+
710
+	/* Init FW admin queue */
711
+	do {
712
+		status = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN);
713
+		if (status)
714
+			return status;
715
+
716
+		status = ice_init_check_adminq(hw);
717
+		if (status != ICE_ERR_AQ_FW_CRITICAL)
718
+			break;
719
+
720
+		ice_debug(hw, ICE_DBG_AQ_MSG,
721
+			  "Retry Admin Queue init due to FW critical error\n");
722
+		ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN);
723
+		msleep(ICE_CTL_Q_ADMIN_INIT_MSEC);
724
+	} while (retry++ < ICE_CTL_Q_ADMIN_INIT_TIMEOUT);
725
+
726
+	if (status)
727
+		return status;
728
+	/* Init Mailbox queue */
729
+	return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX);
730
+}
731
+
732
+/**
733
+ * ice_init_ctrlq_locks - Initialize locks for a control queue
734
+ * @cq: pointer to the control queue
735
+ *
736
+ * Initializes the send and receive queue locks for a given control queue.
737
+ */
738
+static void ice_init_ctrlq_locks(struct ice_ctl_q_info *cq)
739
+{
740
+	mutex_init(&cq->sq_lock);
741
+	mutex_init(&cq->rq_lock);
742
+}
743
+
744
+/**
745
+ * ice_create_all_ctrlq - main initialization routine for all control queues
746
+ * @hw: pointer to the hardware structure
747
+ *
748
+ * Prior to calling this function, the driver *MUST* set the following fields
749
+ * in the cq->structure for all control queues:
750
+ *     - cq->num_sq_entries
751
+ *     - cq->num_rq_entries
752
+ *     - cq->rq_buf_size
753
+ *     - cq->sq_buf_size
754
+ *
755
+ * This function creates all the control queue locks and then calls
756
+ * ice_init_all_ctrlq. It should be called once during driver load. If the
757
+ * driver needs to re-initialize control queues at run time it should call
758
+ * ice_init_all_ctrlq instead.
759
+ */
760
+enum ice_status ice_create_all_ctrlq(struct ice_hw *hw)
761
+{
762
+	ice_init_ctrlq_locks(&hw->adminq);
763
+	ice_init_ctrlq_locks(&hw->mailboxq);
764
+
765
+	return ice_init_all_ctrlq(hw);
766
+}
767
+
768
+/**
769
+ * ice_destroy_ctrlq_locks - Destroy locks for a control queue
770
+ * @cq: pointer to the control queue
771
+ *
772
+ * Destroys the send and receive queue locks for a given control queue.
773
+ */
774
+static void ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq)
775
+{
776
+	mutex_destroy(&cq->sq_lock);
777
+	mutex_destroy(&cq->rq_lock);
778
+}
779
+
780
+/**
781
+ * ice_destroy_all_ctrlq - exit routine for all control queues
782
+ * @hw: pointer to the hardware structure
783
+ *
784
+ * This function shuts down all the control queues and then destroys the
785
+ * control queue locks. It should be called once during driver unload. The
786
+ * driver should call ice_shutdown_all_ctrlq if it needs to shut down and
787
+ * reinitialize control queues, such as in response to a reset event.
788
+ */
789
+void ice_destroy_all_ctrlq(struct ice_hw *hw)
790
+{
791
+	/* shut down all the control queues first */
792
+	ice_shutdown_all_ctrlq(hw);
793
+
794
+	ice_destroy_ctrlq_locks(&hw->adminq);
795
+	ice_destroy_ctrlq_locks(&hw->mailboxq);
739796 }
740797 
741798 /**
....@@ -773,8 +830,54 @@
773830 }
774831 
775832 /**
833
+ * ice_debug_cq
834
+ * @hw: pointer to the hardware structure
835
+ * @desc: pointer to control queue descriptor
836
+ * @buf: pointer to command buffer
837
+ * @buf_len: max length of buf
838
+ *
839
+ * Dumps debug log about control command with descriptor contents.
840
+ */
841
+static void ice_debug_cq(struct ice_hw *hw, void *desc, void *buf, u16 buf_len)
842
+{
843
+	struct ice_aq_desc *cq_desc = (struct ice_aq_desc *)desc;
844
+	u16 len;
845
+
846
+	if (!IS_ENABLED(CONFIG_DYNAMIC_DEBUG) &&
847
+	    !((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask))
848
+		return;
849
+
850
+	if (!desc)
851
+		return;
852
+
853
+	len = le16_to_cpu(cq_desc->datalen);
854
+
855
+	ice_debug(hw, ICE_DBG_AQ_DESC,
856
+		  "CQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n",
857
+		  le16_to_cpu(cq_desc->opcode),
858
+		  le16_to_cpu(cq_desc->flags),
859
+		  le16_to_cpu(cq_desc->datalen), le16_to_cpu(cq_desc->retval));
860
+	ice_debug(hw, ICE_DBG_AQ_DESC, "\tcookie (h,l) 0x%08X 0x%08X\n",
861
+		  le32_to_cpu(cq_desc->cookie_high),
862
+		  le32_to_cpu(cq_desc->cookie_low));
863
+	ice_debug(hw, ICE_DBG_AQ_DESC, "\tparam (0,1)  0x%08X 0x%08X\n",
864
+		  le32_to_cpu(cq_desc->params.generic.param0),
865
+		  le32_to_cpu(cq_desc->params.generic.param1));
866
+	ice_debug(hw, ICE_DBG_AQ_DESC, "\taddr (h,l)   0x%08X 0x%08X\n",
867
+		  le32_to_cpu(cq_desc->params.generic.addr_high),
868
+		  le32_to_cpu(cq_desc->params.generic.addr_low));
869
+	if (buf && cq_desc->datalen != 0) {
870
+		ice_debug(hw, ICE_DBG_AQ_DESC_BUF, "Buffer:\n");
871
+		if (buf_len < len)
872
+			len = buf_len;
873
+
874
+		ice_debug_array(hw, ICE_DBG_AQ_DESC_BUF, 16, 1, (u8 *)buf, len);
875
+	}
876
+}
877
+
878
+/**
776879  * ice_sq_done - check if FW has processed the Admin Send Queue (ATQ)
777
- * @hw: pointer to the hw struct
880
+ * @hw: pointer to the HW struct
778881  * @cq: pointer to the specific Control queue
779882  *
780883  * Returns true if the firmware has processed all descriptors on the
....@@ -790,14 +893,14 @@
790893 
791894 /**
792895  * ice_sq_send_cmd - send command to Control Queue (ATQ)
793
- * @hw: pointer to the hw struct
896
+ * @hw: pointer to the HW struct
794897  * @cq: pointer to the specific Control queue
795898  * @desc: prefilled descriptor describing the command (non DMA mem)
796899  * @buf: buffer to use for indirect commands (or NULL for direct commands)
797900  * @buf_size: size of buffer for indirect commands (or 0 for direct commands)
798901  * @cd: pointer to command details structure
799902  *
800
- * This is the main send command routine for the ATQ.  It runs the q,
903
+ * This is the main send command routine for the ATQ. It runs the queue,
801904  * cleans the queue, etc.
802905  */
803906 enum ice_status
....@@ -858,7 +961,7 @@
858961 
859962 	details = ICE_CTL_Q_DETAILS(cq->sq, cq->sq.next_to_use);
860963 	if (cd)
861
-		memcpy(details, cd, sizeof(*details));
964
+		*details = *cd;
862965 	else
863966 		memset(details, 0, sizeof(*details));
864967 
....@@ -897,10 +1000,10 @@
8971000 	}
8981001 
8991002 	/* Debug desc and buffer */
900
-	ice_debug(hw, ICE_DBG_AQ_MSG,
1003
+	ice_debug(hw, ICE_DBG_AQ_DESC,
9011004 		  "ATQ: Control Send queue desc and buffer:\n");
9021005 
903
-	ice_debug_cq(hw, ICE_DBG_AQ_CMD, (void *)desc_on_ring, buf, buf_size);
1006
+	ice_debug_cq(hw, (void *)desc_on_ring, buf, buf_size);
9041007 
9051008 	(cq->sq.next_to_use)++;
9061009 	if (cq->sq.next_to_use == cq->sq.count)
....@@ -934,7 +1037,8 @@
9341037 		retval = le16_to_cpu(desc->retval);
9351038 		if (retval) {
9361039 			ice_debug(hw, ICE_DBG_AQ_MSG,
937
-				  "Control Send Queue command completed with error 0x%x\n",
1040
+				  "Control Send Queue command 0x%04X completed with error 0x%X\n",
1041
+				  le16_to_cpu(desc->opcode),
9381042 				  retval);
9391043 
9401044 			/* strip off FW internal code */
....@@ -949,7 +1053,7 @@
9491053 	ice_debug(hw, ICE_DBG_AQ_MSG,
9501054 		  "ATQ: desc and buffer writeback:\n");
9511055 
952
-	ice_debug_cq(hw, ICE_DBG_AQ_CMD, (void *)desc, buf, buf_size);
1056
+	ice_debug_cq(hw, (void *)desc, buf, buf_size);
9531057 
9541058 	/* save writeback AQ if requested */
9551059 	if (details->wb_desc)
....@@ -958,9 +1062,15 @@
9581062 
9591063 	/* update the error if time out occurred */
9601064 	if (!cmd_completed) {
961
-		ice_debug(hw, ICE_DBG_AQ_MSG,
962
-			  "Control Send Queue Writeback timeout.\n");
963
-		status = ICE_ERR_AQ_TIMEOUT;
1065
+		if (rd32(hw, cq->rq.len) & cq->rq.len_crit_mask ||
1066
+		    rd32(hw, cq->sq.len) & cq->sq.len_crit_mask) {
1067
+			ice_debug(hw, ICE_DBG_AQ_MSG, "Critical FW error.\n");
1068
+			status = ICE_ERR_AQ_FW_CRITICAL;
1069
+		} else {
1070
+			ice_debug(hw, ICE_DBG_AQ_MSG,
1071
+				  "Control Send Queue Writeback timeout.\n");
1072
+			status = ICE_ERR_AQ_TIMEOUT;
1073
+		}
9641074 	}
9651075 
9661076 sq_send_command_error:
....@@ -985,13 +1095,13 @@
9851095 
9861096 /**
9871097  * ice_clean_rq_elem
988
- * @hw: pointer to the hw struct
1098
+ * @hw: pointer to the HW struct
9891099  * @cq: pointer to the specific Control queue
9901100  * @e: event info from the receive descriptor, includes any buffers
9911101  * @pending: number of events that could be left to process
9921102  *
9931103  * This function cleans one Admin Receive Queue element and returns
994
- * the contents through e.  It can also return how many events are
1104
+ * the contents through e. It can also return how many events are
9951105  * left to process through 'pending'.
9961106  */
9971107 enum ice_status
....@@ -1038,19 +1148,19 @@
10381148 	if (flags & ICE_AQ_FLAG_ERR) {
10391149 		ret_code = ICE_ERR_AQ_ERROR;
10401150 		ice_debug(hw, ICE_DBG_AQ_MSG,
1041
-			  "Control Receive Queue Event received with error 0x%x\n",
1151
+			  "Control Receive Queue Event 0x%04X received with error 0x%X\n",
1152
+			  le16_to_cpu(desc->opcode),
10421153 			  cq->rq_last_status);
10431154 	}
10441155 	memcpy(&e->desc, desc, sizeof(e->desc));
10451156 	datalen = le16_to_cpu(desc->datalen);
1046
-	e->msg_len = min(datalen, e->buf_len);
1157
+	e->msg_len = min_t(u16, datalen, e->buf_len);
10471158 	if (e->msg_buf && e->msg_len)
10481159 		memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va, e->msg_len);
10491160 
1050
-	ice_debug(hw, ICE_DBG_AQ_MSG, "ARQ: desc and buffer:\n");
1161
+	ice_debug(hw, ICE_DBG_AQ_DESC, "ARQ: desc and buffer:\n");
10511162 
1052
-	ice_debug_cq(hw, ICE_DBG_AQ_CMD, (void *)desc, e->msg_buf,
1053
-		     cq->rq_buf_size);
1163
+	ice_debug_cq(hw, (void *)desc, e->msg_buf, cq->rq_buf_size);
10541164 
10551165 	/* Restore the original datalen and buffer address in the desc,
10561166 	 * FW updates datalen to indicate the event message size