disable FB

hc

2024-01-04 1543e317f1da31b75942316931e8f491a8920811

 kernel/drivers/net/ethernet/mellanox/mlx5/core/wq.c
..
..
@@ -34,53 +34,19 @@
34
34
 #include "wq.h"
35
35
 #include "mlx5_core.h"
36
36
 
37
-u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq)
37
+static u32 wq_get_byte_sz(u8 log_sz, u8 log_stride)
38
38
 {
39
-	return (u32)wq->fbc.sz_m1 + 1;
40
-}
41
-
42
-u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq)
43
-{
44
-	return wq->fbc.sz_m1 + 1;
45
-}
46
-
47
-u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq)
48
-{
49
-	return (u32)wq->fbc.sz_m1 + 1;
50
-}
51
-
52
-static u32 mlx5_wq_cyc_get_byte_size(struct mlx5_wq_cyc *wq)
53
-{
54
-	return mlx5_wq_cyc_get_size(wq) << wq->fbc.log_stride;
55
-}
56
-
57
-static u32 mlx5_wq_qp_get_byte_size(struct mlx5_wq_qp *wq)
58
-{
59
-	return mlx5_wq_cyc_get_byte_size(&wq->rq) +
60
-	       mlx5_wq_cyc_get_byte_size(&wq->sq);
61
-}
62
-
63
-static u32 mlx5_cqwq_get_byte_size(struct mlx5_cqwq *wq)
64
-{
65
-	return mlx5_cqwq_get_size(wq) << wq->fbc.log_stride;
66
-}
67
-
68
-static u32 mlx5_wq_ll_get_byte_size(struct mlx5_wq_ll *wq)
69
-{
70
-	return mlx5_wq_ll_get_size(wq) << wq->fbc.log_stride;
39
+	return ((u32)1 << log_sz) << log_stride;
71
40
 }
72
41
 
73
42
 int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
74
43
 		       void *wqc, struct mlx5_wq_cyc *wq,
75
44
 		       struct mlx5_wq_ctrl *wq_ctrl)
76
45
 {
46
+	u8 log_wq_stride = MLX5_GET(wq, wqc, log_wq_stride);
47
+	u8 log_wq_sz     = MLX5_GET(wq, wqc, log_wq_sz);
77
48
 	struct mlx5_frag_buf_ctrl *fbc = &wq->fbc;
78
49
 	int err;
79
-
80
-	mlx5_fill_fbc(MLX5_GET(wq, wqc, log_wq_stride),
81
-		      MLX5_GET(wq, wqc, log_wq_sz),
82
-		      fbc);
83
-	wq->sz    = wq->fbc.sz_m1 + 1;
84
50
 
85
51
 	err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
86
52
 	if (err) {
..
..
@@ -88,15 +54,17 @@
88
54
 		return err;
89
55
 	}
90
56
 
91
-	err = mlx5_frag_buf_alloc_node(mdev, mlx5_wq_cyc_get_byte_size(wq),
57
+	wq->db  = wq_ctrl->db.db;
58
+
59
+	err = mlx5_frag_buf_alloc_node(mdev, wq_get_byte_sz(log_wq_sz, log_wq_stride),
92
60
 				       &wq_ctrl->buf, param->buf_numa_node);
93
61
 	if (err) {
94
62
 		mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n", err);
95
63
 		goto err_db_free;
96
64
 	}
97
65
 
98
-	fbc->frag_buf = wq_ctrl->buf;
99
-	wq->db  = wq_ctrl->db.db;
66
+	mlx5_init_fbc(wq_ctrl->buf.frags, log_wq_stride, log_wq_sz, fbc);
67
+	wq->sz = mlx5_wq_cyc_get_size(wq);
100
68
 
101
69
 	wq_ctrl->mdev = mdev;
102
70
 
..
..
@@ -108,46 +76,44 @@
108
76
 	return err;
109
77
 }
110
78
 
111
-static void mlx5_qp_set_frag_buf(struct mlx5_frag_buf *buf,
112
-				 struct mlx5_wq_qp *qp)
79
+void mlx5_wq_cyc_wqe_dump(struct mlx5_wq_cyc *wq, u16 ix, u8 nstrides)
113
80
 {
114
-	struct mlx5_frag_buf_ctrl *sq_fbc;
115
-	struct mlx5_frag_buf *rqb, *sqb;
81
+	size_t len;
82
+	void *wqe;
116
83
 
117
-	rqb  = &qp->rq.fbc.frag_buf;
118
-	*rqb = *buf;
119
-	rqb->size   = mlx5_wq_cyc_get_byte_size(&qp->rq);
120
-	rqb->npages = DIV_ROUND_UP(rqb->size, PAGE_SIZE);
84
+	if (!net_ratelimit())
85
+		return;
121
86
 
122
-	sq_fbc = &qp->sq.fbc;
123
-	sqb    = &sq_fbc->frag_buf;
124
-	*sqb   = *buf;
125
-	sqb->size   = mlx5_wq_cyc_get_byte_size(&qp->sq);
126
-	sqb->npages = DIV_ROUND_UP(sqb->size, PAGE_SIZE);
127
-	sqb->frags += rqb->npages; /* first part is for the rq */
128
-	if (sq_fbc->strides_offset)
129
-		sqb->frags--;
87
+	nstrides = max_t(u8, nstrides, 1);
88
+
89
+	len = nstrides << wq->fbc.log_stride;
90
+	wqe = mlx5_wq_cyc_get_wqe(wq, ix);
91
+
92
+	pr_info("WQE DUMP: WQ size %d WQ cur size %d, WQE index 0x%x, len: %zu\n",
93
+		mlx5_wq_cyc_get_size(wq), wq->cur_sz, ix, len);
94
+	print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, 16, 1, wqe, len, false);
95
+}
96
+
97
+void mlx5_wq_cyc_reset(struct mlx5_wq_cyc *wq)
98
+{
99
+	wq->wqe_ctr = 0;
100
+	wq->cur_sz = 0;
101
+	mlx5_wq_cyc_update_db_record(wq);
130
102
 }
131
103
 
132
104
 int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
133
105
 		      void *qpc, struct mlx5_wq_qp *wq,
134
106
 		      struct mlx5_wq_ctrl *wq_ctrl)
135
107
 {
136
-	u16 sq_strides_offset;
137
-	u32 rq_pg_remainder;
108
+	u8 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride) + 4;
109
+	u8 log_rq_sz     = MLX5_GET(qpc, qpc, log_rq_size);
110
+	u8 log_sq_stride = ilog2(MLX5_SEND_WQE_BB);
111
+	u8 log_sq_sz     = MLX5_GET(qpc, qpc, log_sq_size);
112
+
113
+	u32 rq_byte_size;
138
114
 	int err;
139
115
 
140
-	mlx5_fill_fbc(MLX5_GET(qpc, qpc, log_rq_stride) + 4,
141
-		      MLX5_GET(qpc, qpc, log_rq_size),
142
-		      &wq->rq.fbc);
143
116
 
144
-	rq_pg_remainder   = mlx5_wq_cyc_get_byte_size(&wq->rq) % PAGE_SIZE;
145
-	sq_strides_offset = rq_pg_remainder / MLX5_SEND_WQE_BB;
146
-
147
-	mlx5_fill_fbc_offset(ilog2(MLX5_SEND_WQE_BB),
148
-			     MLX5_GET(qpc, qpc, log_sq_size),
149
-			     sq_strides_offset,
150
-			     &wq->sq.fbc);
151
117
 
152
118
 	err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
153
119
 	if (err) {
..
..
@@ -155,14 +121,32 @@
155
121
 		return err;
156
122
 	}
157
123
 
158
-	err = mlx5_frag_buf_alloc_node(mdev, mlx5_wq_qp_get_byte_size(wq),
124
+	err = mlx5_frag_buf_alloc_node(mdev,
125
+				       wq_get_byte_sz(log_rq_sz, log_rq_stride) +
126
+				       wq_get_byte_sz(log_sq_sz, log_sq_stride),
159
127
 				       &wq_ctrl->buf, param->buf_numa_node);
160
128
 	if (err) {
161
129
 		mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n", err);
162
130
 		goto err_db_free;
163
131
 	}
164
132
 
165
-	mlx5_qp_set_frag_buf(&wq_ctrl->buf, wq);
133
+	mlx5_init_fbc(wq_ctrl->buf.frags, log_rq_stride, log_rq_sz, &wq->rq.fbc);
134
+
135
+	rq_byte_size = wq_get_byte_sz(log_rq_sz, log_rq_stride);
136
+
137
+	if (rq_byte_size < PAGE_SIZE) {
138
+		/* SQ starts within the same page of the RQ */
139
+		u16 sq_strides_offset = rq_byte_size / MLX5_SEND_WQE_BB;
140
+
141
+		mlx5_init_fbc_offset(wq_ctrl->buf.frags,
142
+				     log_sq_stride, log_sq_sz, sq_strides_offset,
143
+				     &wq->sq.fbc);
144
+	} else {
145
+		u16 rq_npages = rq_byte_size >> PAGE_SHIFT;
146
+
147
+		mlx5_init_fbc(wq_ctrl->buf.frags + rq_npages,
148
+			      log_sq_stride, log_sq_sz, &wq->sq.fbc);
149
+	}
166
150
 
167
151
 	wq->rq.db  = &wq_ctrl->db.db[MLX5_RCV_DBR];
168
152
 	wq->sq.db  = &wq_ctrl->db.db[MLX5_SND_DBR];
..
..
@@ -181,9 +165,10 @@
181
165
 		     void *cqc, struct mlx5_cqwq *wq,
182
166
 		     struct mlx5_wq_ctrl *wq_ctrl)
183
167
 {
168
+	/* CQE_STRIDE_128 and CQE_STRIDE_128_PAD both mean 128B stride */
169
+	u8 log_wq_stride = MLX5_GET(cqc, cqc, cqe_sz) == CQE_STRIDE_64 ? 6 : 7;
170
+	u8 log_wq_sz     = MLX5_GET(cqc, cqc, log_cq_size);
184
171
 	int err;
185
-
186
-	mlx5_core_init_cq_frag_buf(&wq->fbc, cqc);
187
172
 
188
173
 	err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
189
174
 	if (err) {
..
..
@@ -191,7 +176,9 @@
191
176
 		return err;
192
177
 	}
193
178
 
194
-	err = mlx5_frag_buf_alloc_node(mdev, mlx5_cqwq_get_byte_size(wq),
179
+	wq->db  = wq_ctrl->db.db;
180
+
181
+	err = mlx5_frag_buf_alloc_node(mdev, wq_get_byte_sz(log_wq_sz, log_wq_stride),
195
182
 				       &wq_ctrl->buf,
196
183
 				       param->buf_numa_node);
197
184
 	if (err) {
..
..
@@ -200,8 +187,7 @@
200
187
 		goto err_db_free;
201
188
 	}
202
189
 
203
-	wq->fbc.frag_buf = wq_ctrl->buf;
204
-	wq->db  = wq_ctrl->db.db;
190
+	mlx5_init_fbc(wq_ctrl->buf.frags, log_wq_stride, log_wq_sz, &wq->fbc);
205
191
 
206
192
 	wq_ctrl->mdev = mdev;
207
193
 
..
..
@@ -213,18 +199,27 @@
213
199
 	return err;
214
200
 }
215
201
 
202
+static void mlx5_wq_ll_init_list(struct mlx5_wq_ll *wq)
203
+{
204
+	struct mlx5_wqe_srq_next_seg *next_seg;
205
+	int i;
206
+
207
+	for (i = 0; i < wq->fbc.sz_m1; i++) {
208
+		next_seg = mlx5_wq_ll_get_wqe(wq, i);
209
+		next_seg->next_wqe_index = cpu_to_be16(i + 1);
210
+	}
211
+	next_seg = mlx5_wq_ll_get_wqe(wq, i);
212
+	wq->tail_next = &next_seg->next_wqe_index;
213
+}
214
+
216
215
 int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
217
216
 		      void *wqc, struct mlx5_wq_ll *wq,
218
217
 		      struct mlx5_wq_ctrl *wq_ctrl)
219
218
 {
219
+	u8 log_wq_stride = MLX5_GET(wq, wqc, log_wq_stride);
220
+	u8 log_wq_sz     = MLX5_GET(wq, wqc, log_wq_sz);
220
221
 	struct mlx5_frag_buf_ctrl *fbc = &wq->fbc;
221
-	struct mlx5_wqe_srq_next_seg *next_seg;
222
222
 	int err;
223
-	int i;
224
-
225
-	mlx5_fill_fbc(MLX5_GET(wq, wqc, log_wq_stride),
226
-		      MLX5_GET(wq, wqc, log_wq_sz),
227
-		      fbc);
228
223
 
229
224
 	err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
230
225
 	if (err) {
..
..
@@ -232,23 +227,18 @@
232
227
 		return err;
233
228
 	}
234
229
 
235
-	err = mlx5_frag_buf_alloc_node(mdev, mlx5_wq_ll_get_byte_size(wq),
230
+	wq->db  = wq_ctrl->db.db;
231
+
232
+	err = mlx5_frag_buf_alloc_node(mdev, wq_get_byte_sz(log_wq_sz, log_wq_stride),
236
233
 				       &wq_ctrl->buf, param->buf_numa_node);
237
234
 	if (err) {
238
235
 		mlx5_core_warn(mdev, "mlx5_frag_buf_alloc_node() failed, %d\n", err);
239
236
 		goto err_db_free;
240
237
 	}
241
238
 
242
-	wq->fbc.frag_buf = wq_ctrl->buf;
243
-	wq->db  = wq_ctrl->db.db;
239
+	mlx5_init_fbc(wq_ctrl->buf.frags, log_wq_stride, log_wq_sz, fbc);
244
240
 
245
-	for (i = 0; i < fbc->sz_m1; i++) {
246
-		next_seg = mlx5_wq_ll_get_wqe(wq, i);
247
-		next_seg->next_wqe_index = cpu_to_be16(i + 1);
248
-	}
249
-	next_seg = mlx5_wq_ll_get_wqe(wq, i);
250
-	wq->tail_next = &next_seg->next_wqe_index;
251
-
241
+	mlx5_wq_ll_init_list(wq);
252
242
 	wq_ctrl->mdev = mdev;
253
243
 
254
244
 	return 0;
..
..
@@ -259,6 +249,15 @@
259
249
 	return err;
260
250
 }
261
251
 
252
+void mlx5_wq_ll_reset(struct mlx5_wq_ll *wq)
253
+{
254
+	wq->head = 0;
255
+	wq->wqe_ctr = 0;
256
+	wq->cur_sz = 0;
257
+	mlx5_wq_ll_init_list(wq);
258
+	mlx5_wq_ll_update_db_record(wq);
259
+}
260
+
262
261
 void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl)
263
262
 {
264
263
 	mlx5_frag_buf_free(wq_ctrl->mdev, &wq_ctrl->buf);