update kernel to 5.10.198

hc

2024-01-03 2f7c68cb55ecb7331f2381deb497c27155f32faf

 kernel/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
..
..
@@ -30,9 +30,74 @@
30
30
  * SOFTWARE.
31
31
  */
32
32
 
33
+#include "lib/mlx5.h"
33
34
 #include "en.h"
34
-#include "en_accel/ipsec.h"
35
35
 #include "en_accel/tls.h"
36
+#include "en_accel/en_accel.h"
37
+
38
+static unsigned int stats_grps_num(struct mlx5e_priv *priv)
39
+{
40
+	return !priv->profile->stats_grps_num ? 0 :
41
+		priv->profile->stats_grps_num(priv);
42
+}
43
+
44
+unsigned int mlx5e_stats_total_num(struct mlx5e_priv *priv)
45
+{
46
+	mlx5e_stats_grp_t *stats_grps = priv->profile->stats_grps;
47
+	const unsigned int num_stats_grps = stats_grps_num(priv);
48
+	unsigned int total = 0;
49
+	int i;
50
+
51
+	for (i = 0; i < num_stats_grps; i++)
52
+		total += stats_grps[i]->get_num_stats(priv);
53
+
54
+	return total;
55
+}
56
+
57
+void mlx5e_stats_update_ndo_stats(struct mlx5e_priv *priv)
58
+{
59
+	mlx5e_stats_grp_t *stats_grps = priv->profile->stats_grps;
60
+	const unsigned int num_stats_grps = stats_grps_num(priv);
61
+	int i;
62
+
63
+	for (i = num_stats_grps - 1; i >= 0; i--)
64
+		if (stats_grps[i]->update_stats &&
65
+		    stats_grps[i]->update_stats_mask & MLX5E_NDO_UPDATE_STATS)
66
+			stats_grps[i]->update_stats(priv);
67
+}
68
+
69
+void mlx5e_stats_update(struct mlx5e_priv *priv)
70
+{
71
+	mlx5e_stats_grp_t *stats_grps = priv->profile->stats_grps;
72
+	const unsigned int num_stats_grps = stats_grps_num(priv);
73
+	int i;
74
+
75
+	for (i = num_stats_grps - 1; i >= 0; i--)
76
+		if (stats_grps[i]->update_stats)
77
+			stats_grps[i]->update_stats(priv);
78
+}
79
+
80
+void mlx5e_stats_fill(struct mlx5e_priv *priv, u64 *data, int idx)
81
+{
82
+	mlx5e_stats_grp_t *stats_grps = priv->profile->stats_grps;
83
+	const unsigned int num_stats_grps = stats_grps_num(priv);
84
+	int i;
85
+
86
+	for (i = 0; i < num_stats_grps; i++)
87
+		idx = stats_grps[i]->fill_stats(priv, data, idx);
88
+}
89
+
90
+void mlx5e_stats_fill_strings(struct mlx5e_priv *priv, u8 *data)
91
+{
92
+	mlx5e_stats_grp_t *stats_grps = priv->profile->stats_grps;
93
+	const unsigned int num_stats_grps = stats_grps_num(priv);
94
+	int i, idx = 0;
95
+
96
+	for (i = 0; i < num_stats_grps; i++)
97
+		idx = stats_grps[i]->fill_strings(priv, data, idx);
98
+}
99
+
100
+/* Concrete NIC Stats */
36
101
 
37
102
 static const struct counter_desc sw_stats_desc[] = {
38
103
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_packets) },
..
..
@@ -45,10 +110,20 @@
45
110
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tso_inner_bytes) },
46
111
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_added_vlan_packets) },
47
112
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_nop) },
113
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_mpwqe_blks) },
114
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_mpwqe_pkts) },
48
115
 
49
116
 #ifdef CONFIG_MLX5_EN_TLS
117
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_encrypted_packets) },
118
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_encrypted_bytes) },
119
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_ctx) },
50
120
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_ooo) },
121
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_dump_packets) },
122
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_dump_bytes) },
51
123
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_resync_bytes) },
124
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_skip_no_sync_data) },
125
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_drop_no_sync_data) },
126
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_tls_drop_bypass_req) },
52
127
 #endif
53
128
 
54
129
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_lro_packets) },
..
..
@@ -64,6 +139,9 @@
64
139
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_drop) },
65
140
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_redirect) },
66
141
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_xmit) },
142
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_mpwqe) },
143
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_inlnw) },
144
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_nops) },
67
145
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) },
68
146
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_err) },
69
147
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_cqe) },
..
..
@@ -78,6 +156,9 @@
78
156
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_wake) },
79
157
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_cqe_err) },
80
158
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_xmit) },
159
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_mpwqe) },
160
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_inlnw) },
161
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_nops) },
81
162
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_full) },
82
163
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_err) },
83
164
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_cqes) },
..
..
@@ -88,28 +169,68 @@
88
169
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_buff_alloc_err) },
89
170
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_blks) },
90
171
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_pkts) },
91
-	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_page_reuse) },
92
172
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_reuse) },
93
173
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) },
94
174
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },
95
175
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) },
96
176
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_waive) },
97
177
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_congst_umr) },
178
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_arfs_err) },
179
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_recover) },
180
+#ifdef CONFIG_MLX5_EN_TLS
181
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_decrypted_packets) },
182
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_decrypted_bytes) },
183
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_ctx) },
184
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_del) },
185
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_pkt) },
186
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_start) },
187
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_end) },
188
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_req_skip) },
189
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_res_ok) },
190
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_resync_res_skip) },
191
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_tls_err) },
192
+#endif
98
193
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_events) },
99
194
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_poll) },
100
195
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_arm) },
101
196
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_aff_change) },
197
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_force_irq) },
102
198
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_eq_rearm) },
199
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_packets) },
200
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_bytes) },
201
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_csum_complete) },
202
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_csum_unnecessary) },
203
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_csum_unnecessary_inner) },
204
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_csum_none) },
205
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_ecn_mark) },
206
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_removed_vlan_packets) },
207
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_xdp_drop) },
208
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_xdp_redirect) },
209
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_wqe_err) },
210
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_mpwqe_filler_cqes) },
211
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_mpwqe_filler_strides) },
212
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_oversize_pkts_sw_drop) },
213
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_buff_alloc_err) },
214
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_cqe_compress_blks) },
215
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_cqe_compress_pkts) },
216
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_congst_umr) },
217
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xsk_arfs_err) },
218
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_xmit) },
219
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_mpwqe) },
220
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_inlnw) },
221
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_full) },
222
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_err) },
223
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xsk_cqes) },
103
224
 };
104
225
 
105
226
 #define NUM_SW_COUNTERS			ARRAY_SIZE(sw_stats_desc)
106
227
 
107
-static int mlx5e_grp_sw_get_num_stats(struct mlx5e_priv *priv)
228
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(sw)
108
229
 {
109
230
 	return NUM_SW_COUNTERS;
110
231
 }
111
232
 
112
-static int mlx5e_grp_sw_fill_strings(struct mlx5e_priv *priv, u8 *data, int idx)
233
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(sw)
113
234
 {
114
235
 	int i;
115
236
 
..
..
@@ -118,7 +239,7 @@
118
239
 	return idx;
119
240
 }
120
241
 
121
-static int mlx5e_grp_sw_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
242
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(sw)
122
243
 {
123
244
 	int i;
124
245
 
..
..
@@ -127,18 +248,20 @@
127
248
 	return idx;
128
249
 }
129
250
 
130
-void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
251
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw)
131
252
 {
132
-	struct mlx5e_sw_stats temp, *s = &temp;
253
+	struct mlx5e_sw_stats *s = &priv->stats.sw;
133
254
 	int i;
134
255
 
135
256
 	memset(s, 0, sizeof(*s));
136
257
 
137
-	for (i = 0; i < priv->profile->max_nch(priv->mdev); i++) {
258
+	for (i = 0; i < priv->max_nch; i++) {
138
259
 		struct mlx5e_channel_stats *channel_stats =
139
260
 			&priv->channel_stats[i];
140
261
 		struct mlx5e_xdpsq_stats *xdpsq_red_stats = &channel_stats->xdpsq;
141
262
 		struct mlx5e_xdpsq_stats *xdpsq_stats = &channel_stats->rq_xdpsq;
263
+		struct mlx5e_xdpsq_stats *xsksq_stats = &channel_stats->xsksq;
264
+		struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq;
142
265
 		struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
143
266
 		struct mlx5e_ch_stats *ch_stats = &channel_stats->ch;
144
267
 		int j;
..
..
@@ -158,6 +281,9 @@
158
281
 		s->rx_xdp_drop     += rq_stats->xdp_drop;
159
282
 		s->rx_xdp_redirect += rq_stats->xdp_redirect;
160
283
 		s->rx_xdp_tx_xmit  += xdpsq_stats->xmit;
284
+		s->rx_xdp_tx_mpwqe += xdpsq_stats->mpwqe;
285
+		s->rx_xdp_tx_inlnw += xdpsq_stats->inlnw;
286
+		s->rx_xdp_tx_nops  += xdpsq_stats->nops;
161
287
 		s->rx_xdp_tx_full  += xdpsq_stats->full;
162
288
 		s->rx_xdp_tx_err   += xdpsq_stats->err;
163
289
 		s->rx_xdp_tx_cqe   += xdpsq_stats->cqes;
..
..
@@ -168,23 +294,67 @@
168
294
 		s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
169
295
 		s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
170
296
 		s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
171
-		s->rx_page_reuse  += rq_stats->page_reuse;
172
297
 		s->rx_cache_reuse += rq_stats->cache_reuse;
173
298
 		s->rx_cache_full  += rq_stats->cache_full;
174
299
 		s->rx_cache_empty += rq_stats->cache_empty;
175
300
 		s->rx_cache_busy  += rq_stats->cache_busy;
176
301
 		s->rx_cache_waive += rq_stats->cache_waive;
177
302
 		s->rx_congst_umr  += rq_stats->congst_umr;
303
+		s->rx_arfs_err    += rq_stats->arfs_err;
304
+		s->rx_recover     += rq_stats->recover;
305
+#ifdef CONFIG_MLX5_EN_TLS
306
+		s->rx_tls_decrypted_packets += rq_stats->tls_decrypted_packets;
307
+		s->rx_tls_decrypted_bytes   += rq_stats->tls_decrypted_bytes;
308
+		s->rx_tls_ctx               += rq_stats->tls_ctx;
309
+		s->rx_tls_del               += rq_stats->tls_del;
310
+		s->rx_tls_resync_req_pkt    += rq_stats->tls_resync_req_pkt;
311
+		s->rx_tls_resync_req_start  += rq_stats->tls_resync_req_start;
312
+		s->rx_tls_resync_req_end    += rq_stats->tls_resync_req_end;
313
+		s->rx_tls_resync_req_skip   += rq_stats->tls_resync_req_skip;
314
+		s->rx_tls_resync_res_ok     += rq_stats->tls_resync_res_ok;
315
+		s->rx_tls_resync_res_skip   += rq_stats->tls_resync_res_skip;
316
+		s->rx_tls_err               += rq_stats->tls_err;
317
+#endif
178
318
 		s->ch_events      += ch_stats->events;
179
319
 		s->ch_poll        += ch_stats->poll;
180
320
 		s->ch_arm         += ch_stats->arm;
181
321
 		s->ch_aff_change  += ch_stats->aff_change;
322
+		s->ch_force_irq   += ch_stats->force_irq;
182
323
 		s->ch_eq_rearm    += ch_stats->eq_rearm;
183
324
 		/* xdp redirect */
184
325
 		s->tx_xdp_xmit    += xdpsq_red_stats->xmit;
326
+		s->tx_xdp_mpwqe   += xdpsq_red_stats->mpwqe;
327
+		s->tx_xdp_inlnw   += xdpsq_red_stats->inlnw;
328
+		s->tx_xdp_nops	  += xdpsq_red_stats->nops;
185
329
 		s->tx_xdp_full    += xdpsq_red_stats->full;
186
330
 		s->tx_xdp_err     += xdpsq_red_stats->err;
187
331
 		s->tx_xdp_cqes    += xdpsq_red_stats->cqes;
332
+		/* AF_XDP zero-copy */
333
+		s->rx_xsk_packets                += xskrq_stats->packets;
334
+		s->rx_xsk_bytes                  += xskrq_stats->bytes;
335
+		s->rx_xsk_csum_complete          += xskrq_stats->csum_complete;
336
+		s->rx_xsk_csum_unnecessary       += xskrq_stats->csum_unnecessary;
337
+		s->rx_xsk_csum_unnecessary_inner += xskrq_stats->csum_unnecessary_inner;
338
+		s->rx_xsk_csum_none              += xskrq_stats->csum_none;
339
+		s->rx_xsk_ecn_mark               += xskrq_stats->ecn_mark;
340
+		s->rx_xsk_removed_vlan_packets   += xskrq_stats->removed_vlan_packets;
341
+		s->rx_xsk_xdp_drop               += xskrq_stats->xdp_drop;
342
+		s->rx_xsk_xdp_redirect           += xskrq_stats->xdp_redirect;
343
+		s->rx_xsk_wqe_err                += xskrq_stats->wqe_err;
344
+		s->rx_xsk_mpwqe_filler_cqes      += xskrq_stats->mpwqe_filler_cqes;
345
+		s->rx_xsk_mpwqe_filler_strides   += xskrq_stats->mpwqe_filler_strides;
346
+		s->rx_xsk_oversize_pkts_sw_drop  += xskrq_stats->oversize_pkts_sw_drop;
347
+		s->rx_xsk_buff_alloc_err         += xskrq_stats->buff_alloc_err;
348
+		s->rx_xsk_cqe_compress_blks      += xskrq_stats->cqe_compress_blks;
349
+		s->rx_xsk_cqe_compress_pkts      += xskrq_stats->cqe_compress_pkts;
350
+		s->rx_xsk_congst_umr             += xskrq_stats->congst_umr;
351
+		s->rx_xsk_arfs_err               += xskrq_stats->arfs_err;
352
+		s->tx_xsk_xmit                   += xsksq_stats->xmit;
353
+		s->tx_xsk_mpwqe                  += xsksq_stats->mpwqe;
354
+		s->tx_xsk_inlnw                  += xsksq_stats->inlnw;
355
+		s->tx_xsk_full                   += xsksq_stats->full;
356
+		s->tx_xsk_err                    += xsksq_stats->err;
357
+		s->tx_xsk_cqes                   += xsksq_stats->cqes;
188
358
 
189
359
 		for (j = 0; j < priv->max_opened_tc; j++) {
190
360
 			struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
..
..
@@ -197,6 +367,8 @@
197
367
 			s->tx_tso_inner_bytes	+= sq_stats->tso_inner_bytes;
198
368
 			s->tx_added_vlan_packets += sq_stats->added_vlan_packets;
199
369
 			s->tx_nop               += sq_stats->nop;
370
+			s->tx_mpwqe_blks        += sq_stats->mpwqe_blks;
371
+			s->tx_mpwqe_pkts        += sq_stats->mpwqe_pkts;
200
372
 			s->tx_queue_stopped	+= sq_stats->stopped;
201
373
 			s->tx_queue_wake	+= sq_stats->wake;
202
374
 			s->tx_queue_dropped	+= sq_stats->dropped;
..
..
@@ -207,8 +379,16 @@
207
379
 			s->tx_csum_none		+= sq_stats->csum_none;
208
380
 			s->tx_csum_partial	+= sq_stats->csum_partial;
209
381
 #ifdef CONFIG_MLX5_EN_TLS
210
-			s->tx_tls_ooo		+= sq_stats->tls_ooo;
211
-			s->tx_tls_resync_bytes	+= sq_stats->tls_resync_bytes;
382
+			s->tx_tls_encrypted_packets += sq_stats->tls_encrypted_packets;
383
+			s->tx_tls_encrypted_bytes   += sq_stats->tls_encrypted_bytes;
384
+			s->tx_tls_ctx               += sq_stats->tls_ctx;
385
+			s->tx_tls_ooo               += sq_stats->tls_ooo;
386
+			s->tx_tls_dump_bytes        += sq_stats->tls_dump_bytes;
387
+			s->tx_tls_dump_packets      += sq_stats->tls_dump_packets;
388
+			s->tx_tls_resync_bytes      += sq_stats->tls_resync_bytes;
389
+			s->tx_tls_skip_no_sync_data += sq_stats->tls_skip_no_sync_data;
390
+			s->tx_tls_drop_no_sync_data += sq_stats->tls_drop_no_sync_data;
391
+			s->tx_tls_drop_bypass_req   += sq_stats->tls_drop_bypass_req;
212
392
 #endif
213
393
 			s->tx_cqes		+= sq_stats->cqes;
214
394
 
..
..
@@ -216,8 +396,6 @@
216
396
 			barrier();
217
397
 		}
218
398
 	}
219
-
220
-	memcpy(&priv->stats.sw, s, sizeof(*s));
221
399
 }
222
400
 
223
401
 static const struct counter_desc q_stats_desc[] = {
..
..
@@ -231,7 +409,7 @@
231
409
 #define NUM_Q_COUNTERS			ARRAY_SIZE(q_stats_desc)
232
410
 #define NUM_DROP_RQ_COUNTERS		ARRAY_SIZE(drop_rq_stats_desc)
233
411
 
234
-static int mlx5e_grp_q_get_num_stats(struct mlx5e_priv *priv)
412
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(qcnt)
235
413
 {
236
414
 	int num_stats = 0;
237
415
 
..
..
@@ -244,7 +422,7 @@
244
422
 	return num_stats;
245
423
 }
246
424
 
247
-static int mlx5e_grp_q_fill_strings(struct mlx5e_priv *priv, u8 *data, int idx)
425
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(qcnt)
248
426
 {
249
427
 	int i;
250
428
 
..
..
@@ -259,7 +437,7 @@
259
437
 	return idx;
260
438
 }
261
439
 
262
-static int mlx5e_grp_q_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
440
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(qcnt)
263
441
 {
264
442
 	int i;
265
443
 
..
..
@@ -272,80 +450,97 @@
272
450
 	return idx;
273
451
 }
274
452
 
275
-static void mlx5e_grp_q_update_stats(struct mlx5e_priv *priv)
453
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(qcnt)
276
454
 {
277
455
 	struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
278
-	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)];
456
+	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
457
+	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
458
+	int ret;
279
459
 
280
-	if (priv->q_counter &&
281
-	    !mlx5_core_query_q_counter(priv->mdev, priv->q_counter, 0, out,
282
-				       sizeof(out)))
283
-		qcnt->rx_out_of_buffer = MLX5_GET(query_q_counter_out,
284
-						  out, out_of_buffer);
285
-	if (priv->drop_rq_q_counter &&
286
-	    !mlx5_core_query_q_counter(priv->mdev, priv->drop_rq_q_counter, 0,
287
-				       out, sizeof(out)))
288
-		qcnt->rx_if_down_packets = MLX5_GET(query_q_counter_out, out,
289
-						    out_of_buffer);
460
+	MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
461
+
462
+	if (priv->q_counter) {
463
+		MLX5_SET(query_q_counter_in, in, counter_set_id,
464
+			 priv->q_counter);
465
+		ret = mlx5_cmd_exec_inout(priv->mdev, query_q_counter, in, out);
466
+		if (!ret)
467
+			qcnt->rx_out_of_buffer = MLX5_GET(query_q_counter_out,
468
+							  out, out_of_buffer);
469
+	}
470
+
471
+	if (priv->drop_rq_q_counter) {
472
+		MLX5_SET(query_q_counter_in, in, counter_set_id,
473
+			 priv->drop_rq_q_counter);
474
+		ret = mlx5_cmd_exec_inout(priv->mdev, query_q_counter, in, out);
475
+		if (!ret)
476
+			qcnt->rx_if_down_packets = MLX5_GET(query_q_counter_out,
477
+							    out, out_of_buffer);
478
+	}
290
479
 }
291
480
 
292
481
 #define VNIC_ENV_OFF(c) MLX5_BYTE_OFF(query_vnic_env_out, c)
293
-static const struct counter_desc vnic_env_stats_desc[] = {
482
+static const struct counter_desc vnic_env_stats_steer_desc[] = {
294
483
 	{ "rx_steer_missed_packets",
295
484
 		VNIC_ENV_OFF(vport_env.nic_receive_steering_discard) },
296
485
 };
297
486
 
298
-#define NUM_VNIC_ENV_COUNTERS		ARRAY_SIZE(vnic_env_stats_desc)
487
+static const struct counter_desc vnic_env_stats_dev_oob_desc[] = {
488
+	{ "dev_internal_queue_oob",
489
+		VNIC_ENV_OFF(vport_env.internal_rq_out_of_buffer) },
490
+};
299
491
 
300
-static int mlx5e_grp_vnic_env_get_num_stats(struct mlx5e_priv *priv)
492
+#define NUM_VNIC_ENV_STEER_COUNTERS(dev) \
493
+	(MLX5_CAP_GEN(dev, nic_receive_steering_discard) ? \
494
+	 ARRAY_SIZE(vnic_env_stats_steer_desc) : 0)
495
+#define NUM_VNIC_ENV_DEV_OOB_COUNTERS(dev) \
496
+	(MLX5_CAP_GEN(dev, vnic_env_int_rq_oob) ? \
497
+	 ARRAY_SIZE(vnic_env_stats_dev_oob_desc) : 0)
498
+
499
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vnic_env)
301
500
 {
302
-	return MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard) ?
303
-		NUM_VNIC_ENV_COUNTERS : 0;
501
+	return NUM_VNIC_ENV_STEER_COUNTERS(priv->mdev) +
502
+		NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev);
304
503
 }
305
504
 
306
-static int mlx5e_grp_vnic_env_fill_strings(struct mlx5e_priv *priv, u8 *data,
307
-					   int idx)
505
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vnic_env)
308
506
 {
309
507
 	int i;
310
508
 
311
-	if (!MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard))
312
-		return idx;
313
-
314
-	for (i = 0; i < NUM_VNIC_ENV_COUNTERS; i++)
509
+	for (i = 0; i < NUM_VNIC_ENV_STEER_COUNTERS(priv->mdev); i++)
315
510
 		strcpy(data + (idx++) * ETH_GSTRING_LEN,
316
-		       vnic_env_stats_desc[i].format);
511
+		       vnic_env_stats_steer_desc[i].format);
512
+
513
+	for (i = 0; i < NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev); i++)
514
+		strcpy(data + (idx++) * ETH_GSTRING_LEN,
515
+		       vnic_env_stats_dev_oob_desc[i].format);
317
516
 	return idx;
318
517
 }
319
518
 
320
-static int mlx5e_grp_vnic_env_fill_stats(struct mlx5e_priv *priv, u64 *data,
321
-					 int idx)
519
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vnic_env)
322
520
 {
323
521
 	int i;
324
522
 
325
-	if (!MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard))
326
-		return idx;
327
-
328
-	for (i = 0; i < NUM_VNIC_ENV_COUNTERS; i++)
523
+	for (i = 0; i < NUM_VNIC_ENV_STEER_COUNTERS(priv->mdev); i++)
329
524
 		data[idx++] = MLX5E_READ_CTR64_BE(priv->stats.vnic.query_vnic_env_out,
330
-						  vnic_env_stats_desc, i);
525
+						  vnic_env_stats_steer_desc, i);
526
+
527
+	for (i = 0; i < NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev); i++)
528
+		data[idx++] = MLX5E_READ_CTR32_BE(priv->stats.vnic.query_vnic_env_out,
529
+						  vnic_env_stats_dev_oob_desc, i);
331
530
 	return idx;
332
531
 }
333
532
 
334
-static void mlx5e_grp_vnic_env_update_stats(struct mlx5e_priv *priv)
533
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vnic_env)
335
534
 {
336
535
 	u32 *out = (u32 *)priv->stats.vnic.query_vnic_env_out;
337
-	int outlen = MLX5_ST_SZ_BYTES(query_vnic_env_out);
338
-	u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {0};
536
+	u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {};
339
537
 	struct mlx5_core_dev *mdev = priv->mdev;
340
538
 
341
-	if (!MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard))
539
+	if (!mlx5e_stats_grp_vnic_env_num_stats(priv))
342
540
 		return;
343
541
 
344
-	MLX5_SET(query_vnic_env_in, in, opcode,
345
-		 MLX5_CMD_OP_QUERY_VNIC_ENV);
346
-	MLX5_SET(query_vnic_env_in, in, op_mod, 0);
347
-	MLX5_SET(query_vnic_env_in, in, other_vport, 0);
348
-	mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
542
+	MLX5_SET(query_vnic_env_in, in, opcode, MLX5_CMD_OP_QUERY_VNIC_ENV);
543
+	mlx5_cmd_exec_inout(mdev, query_vnic_env, in, out);
349
544
 }
350
545
 
351
546
 #define VPORT_COUNTER_OFF(c) MLX5_BYTE_OFF(query_vport_counter_out, c)
..
..
@@ -394,13 +589,12 @@
394
589
 
395
590
 #define NUM_VPORT_COUNTERS		ARRAY_SIZE(vport_stats_desc)
396
591
 
397
-static int mlx5e_grp_vport_get_num_stats(struct mlx5e_priv *priv)
592
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vport)
398
593
 {
399
594
 	return NUM_VPORT_COUNTERS;
400
595
 }
401
596
 
402
-static int mlx5e_grp_vport_fill_strings(struct mlx5e_priv *priv, u8 *data,
403
-					int idx)
597
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vport)
404
598
 {
405
599
 	int i;
406
600
 
..
..
@@ -409,8 +603,7 @@
409
603
 	return idx;
410
604
 }
411
605
 
412
-static int mlx5e_grp_vport_fill_stats(struct mlx5e_priv *priv, u64 *data,
413
-				      int idx)
606
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vport)
414
607
 {
415
608
 	int i;
416
609
 
..
..
@@ -420,17 +613,14 @@
420
613
 	return idx;
421
614
 }
422
615
 
423
-static void mlx5e_grp_vport_update_stats(struct mlx5e_priv *priv)
616
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vport)
424
617
 {
425
-	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
426
618
 	u32 *out = (u32 *)priv->stats.vport.query_vport_out;
427
-	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
619
+	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {};
428
620
 	struct mlx5_core_dev *mdev = priv->mdev;
429
621
 
430
622
 	MLX5_SET(query_vport_counter_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_COUNTER);
431
-	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
432
-	MLX5_SET(query_vport_counter_in, in, other_vport, 0);
433
-	mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
623
+	mlx5_cmd_exec_inout(mdev, query_vport_counter, in, out);
434
624
 }
435
625
 
436
626
 #define PPORT_802_3_OFF(c) \
..
..
@@ -459,13 +649,12 @@
459
649
 
460
650
 #define NUM_PPORT_802_3_COUNTERS	ARRAY_SIZE(pport_802_3_stats_desc)
461
651
 
462
-static int mlx5e_grp_802_3_get_num_stats(struct mlx5e_priv *priv)
652
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(802_3)
463
653
 {
464
654
 	return NUM_PPORT_802_3_COUNTERS;
465
655
 }
466
656
 
467
-static int mlx5e_grp_802_3_fill_strings(struct mlx5e_priv *priv, u8 *data,
468
-					int idx)
657
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(802_3)
469
658
 {
470
659
 	int i;
471
660
 
..
..
@@ -474,8 +663,7 @@
474
663
 	return idx;
475
664
 }
476
665
 
477
-static int mlx5e_grp_802_3_fill_stats(struct mlx5e_priv *priv, u64 *data,
478
-				      int idx)
666
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(802_3)
479
667
 {
480
668
 	int i;
481
669
 
..
..
@@ -485,7 +673,10 @@
485
673
 	return idx;
486
674
 }
487
675
 
488
-static void mlx5e_grp_802_3_update_stats(struct mlx5e_priv *priv)
676
+#define MLX5_BASIC_PPCNT_SUPPORTED(mdev) \
677
+	(MLX5_CAP_GEN(mdev, pcam_reg) ? MLX5_CAP_PCAM_REG(mdev, ppcnt) : 1)
678
+
679
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(802_3)
489
680
 {
490
681
 	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
491
682
 	struct mlx5_core_dev *mdev = priv->mdev;
..
..
@@ -493,10 +684,42 @@
493
684
 	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
494
685
 	void *out;
495
686
 
687
+	if (!MLX5_BASIC_PPCNT_SUPPORTED(mdev))
688
+		return;
689
+
496
690
 	MLX5_SET(ppcnt_reg, in, local_port, 1);
497
691
 	out = pstats->IEEE_802_3_counters;
498
692
 	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
499
693
 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
694
+}
695
+
696
+#define MLX5E_READ_CTR64_BE_F(ptr, c)			\
697
+	be64_to_cpu(*(__be64 *)((char *)ptr +		\
698
+		MLX5_BYTE_OFF(ppcnt_reg,		\
699
+			counter_set.eth_802_3_cntrs_grp_data_layout.c##_high)))
700
+
701
+void mlx5e_stats_pause_get(struct mlx5e_priv *priv,
702
+			   struct ethtool_pause_stats *pause_stats)
703
+{
704
+	u32 ppcnt_ieee_802_3[MLX5_ST_SZ_DW(ppcnt_reg)];
705
+	struct mlx5_core_dev *mdev = priv->mdev;
706
+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
707
+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
708
+
709
+	if (!MLX5_BASIC_PPCNT_SUPPORTED(mdev))
710
+		return;
711
+
712
+	MLX5_SET(ppcnt_reg, in, local_port, 1);
713
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
714
+	mlx5_core_access_reg(mdev, in, sz, ppcnt_ieee_802_3,
715
+			     sz, MLX5_REG_PPCNT, 0, 0);
716
+
717
+	pause_stats->tx_pause_frames =
718
+		MLX5E_READ_CTR64_BE_F(ppcnt_ieee_802_3,
719
+				      a_pause_mac_ctrl_frames_transmitted);
720
+	pause_stats->rx_pause_frames =
721
+		MLX5E_READ_CTR64_BE_F(ppcnt_ieee_802_3,
722
+				      a_pause_mac_ctrl_frames_received);
500
723
 }
501
724
 
502
725
 #define PPORT_2863_OFF(c) \
..
..
@@ -510,13 +733,12 @@
510
733
 
511
734
 #define NUM_PPORT_2863_COUNTERS		ARRAY_SIZE(pport_2863_stats_desc)
512
735
 
513
-static int mlx5e_grp_2863_get_num_stats(struct mlx5e_priv *priv)
736
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(2863)
514
737
 {
515
738
 	return NUM_PPORT_2863_COUNTERS;
516
739
 }
517
740
 
518
-static int mlx5e_grp_2863_fill_strings(struct mlx5e_priv *priv, u8 *data,
519
-				       int idx)
741
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(2863)
520
742
 {
521
743
 	int i;
522
744
 
..
..
@@ -525,8 +747,7 @@
525
747
 	return idx;
526
748
 }
527
749
 
528
-static int mlx5e_grp_2863_fill_stats(struct mlx5e_priv *priv, u64 *data,
529
-				     int idx)
750
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(2863)
530
751
 {
531
752
 	int i;
532
753
 
..
..
@@ -536,7 +757,7 @@
536
757
 	return idx;
537
758
 }
538
759
 
539
-static void mlx5e_grp_2863_update_stats(struct mlx5e_priv *priv)
760
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(2863)
540
761
 {
541
762
 	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
542
763
 	struct mlx5_core_dev *mdev = priv->mdev;
..
..
@@ -571,13 +792,12 @@
571
792
 
572
793
 #define NUM_PPORT_2819_COUNTERS		ARRAY_SIZE(pport_2819_stats_desc)
573
794
 
574
-static int mlx5e_grp_2819_get_num_stats(struct mlx5e_priv *priv)
795
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(2819)
575
796
 {
576
797
 	return NUM_PPORT_2819_COUNTERS;
577
798
 }
578
799
 
579
-static int mlx5e_grp_2819_fill_strings(struct mlx5e_priv *priv, u8 *data,
580
-				       int idx)
800
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(2819)
581
801
 {
582
802
 	int i;
583
803
 
..
..
@@ -586,8 +806,7 @@
586
806
 	return idx;
587
807
 }
588
808
 
589
-static int mlx5e_grp_2819_fill_stats(struct mlx5e_priv *priv, u64 *data,
590
-				     int idx)
809
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(2819)
591
810
 {
592
811
 	int i;
593
812
 
..
..
@@ -597,13 +816,16 @@
597
816
 	return idx;
598
817
 }
599
818
 
600
-static void mlx5e_grp_2819_update_stats(struct mlx5e_priv *priv)
819
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(2819)
601
820
 {
602
821
 	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
603
822
 	struct mlx5_core_dev *mdev = priv->mdev;
604
823
 	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
605
824
 	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
606
825
 	void *out;
826
+
827
+	if (!MLX5_BASIC_PPCNT_SUPPORTED(mdev))
828
+		return;
607
829
 
608
830
 	MLX5_SET(ppcnt_reg, in, local_port, 1);
609
831
 	out = pstats->RFC_2819_counters;
..
..
@@ -619,50 +841,85 @@
619
841
 	{ "rx_corrected_bits_phy", PPORT_PHY_STATISTICAL_OFF(phy_corrected_bits) },
620
842
 };
621
843
 
622
-#define NUM_PPORT_PHY_STATISTICAL_COUNTERS ARRAY_SIZE(pport_phy_statistical_stats_desc)
844
+static const struct counter_desc
845
+pport_phy_statistical_err_lanes_stats_desc[] = {
846
+	{ "rx_err_lane_0_phy", PPORT_PHY_STATISTICAL_OFF(phy_corrected_bits_lane0) },
847
+	{ "rx_err_lane_1_phy", PPORT_PHY_STATISTICAL_OFF(phy_corrected_bits_lane1) },
848
+	{ "rx_err_lane_2_phy", PPORT_PHY_STATISTICAL_OFF(phy_corrected_bits_lane2) },
849
+	{ "rx_err_lane_3_phy", PPORT_PHY_STATISTICAL_OFF(phy_corrected_bits_lane3) },
850
+};
623
851
 
624
-static int mlx5e_grp_phy_get_num_stats(struct mlx5e_priv *priv)
852
+#define NUM_PPORT_PHY_STATISTICAL_COUNTERS \
853
+	ARRAY_SIZE(pport_phy_statistical_stats_desc)
854
+#define NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS \
855
+	ARRAY_SIZE(pport_phy_statistical_err_lanes_stats_desc)
856
+
857
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(phy)
625
858
 {
859
+	struct mlx5_core_dev *mdev = priv->mdev;
860
+	int num_stats;
861
+
626
862
 	/* "1" for link_down_events special counter */
627
-	return MLX5_CAP_PCAM_FEATURE((priv)->mdev, ppcnt_statistical_group) ?
628
-		NUM_PPORT_PHY_STATISTICAL_COUNTERS + 1 : 1;
863
+	num_stats = 1;
864
+
865
+	num_stats += MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group) ?
866
+		     NUM_PPORT_PHY_STATISTICAL_COUNTERS : 0;
867
+
868
+	num_stats += MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters) ?
869
+		     NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS : 0;
870
+
871
+	return num_stats;
629
872
 }
630
873
 
631
-static int mlx5e_grp_phy_fill_strings(struct mlx5e_priv *priv, u8 *data,
632
-				      int idx)
874
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(phy)
633
875
 {
876
+	struct mlx5_core_dev *mdev = priv->mdev;
634
877
 	int i;
635
878
 
636
879
 	strcpy(data + (idx++) * ETH_GSTRING_LEN, "link_down_events_phy");
637
880
 
638
-	if (!MLX5_CAP_PCAM_FEATURE((priv)->mdev, ppcnt_statistical_group))
881
+	if (!MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group))
639
882
 		return idx;
640
883
 
641
884
 	for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++)
642
885
 		strcpy(data + (idx++) * ETH_GSTRING_LEN,
643
886
 		       pport_phy_statistical_stats_desc[i].format);
887
+
888
+	if (MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters))
889
+		for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS; i++)
890
+			strcpy(data + (idx++) * ETH_GSTRING_LEN,
891
+			       pport_phy_statistical_err_lanes_stats_desc[i].format);
892
+
644
893
 	return idx;
645
894
 }
646
895
 
647
-static int mlx5e_grp_phy_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
896
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(phy)
648
897
 {
898
+	struct mlx5_core_dev *mdev = priv->mdev;
649
899
 	int i;
650
900
 
651
901
 	/* link_down_events_phy has special handling since it is not stored in __be64 format */
652
902
 	data[idx++] = MLX5_GET(ppcnt_reg, priv->stats.pport.phy_counters,
653
903
 			       counter_set.phys_layer_cntrs.link_down_events);
654
904
 
655
-	if (!MLX5_CAP_PCAM_FEATURE((priv)->mdev, ppcnt_statistical_group))
905
+	if (!MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group))
656
906
 		return idx;
657
907
 
658
908
 	for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++)
659
909
 		data[idx++] =
660
910
 			MLX5E_READ_CTR64_BE(&priv->stats.pport.phy_statistical_counters,
661
911
 					    pport_phy_statistical_stats_desc, i);
912
+
913
+	if (MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters))
914
+		for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS; i++)
915
+			data[idx++] =
916
+				MLX5E_READ_CTR64_BE(&priv->stats.pport.phy_statistical_counters,
917
+						    pport_phy_statistical_err_lanes_stats_desc,
918
+						    i);
662
919
 	return idx;
663
920
 }
664
921
 
665
-static void mlx5e_grp_phy_update_stats(struct mlx5e_priv *priv)
922
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(phy)
666
923
 {
667
924
 	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
668
925
 	struct mlx5_core_dev *mdev = priv->mdev;
..
..
@@ -692,7 +949,7 @@
692
949
 
693
950
 #define NUM_PPORT_ETH_EXT_COUNTERS	ARRAY_SIZE(pport_eth_ext_stats_desc)
694
951
 
695
-static int mlx5e_grp_eth_ext_get_num_stats(struct mlx5e_priv *priv)
952
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(eth_ext)
696
953
 {
697
954
 	if (MLX5_CAP_PCAM_FEATURE((priv)->mdev, rx_buffer_fullness_counters))
698
955
 		return NUM_PPORT_ETH_EXT_COUNTERS;
..
..
@@ -700,8 +957,7 @@
700
957
 	return 0;
701
958
 }
702
959
 
703
-static int mlx5e_grp_eth_ext_fill_strings(struct mlx5e_priv *priv, u8 *data,
704
-					  int idx)
960
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(eth_ext)
705
961
 {
706
962
 	int i;
707
963
 
..
..
@@ -712,8 +968,7 @@
712
968
 	return idx;
713
969
 }
714
970
 
715
-static int mlx5e_grp_eth_ext_fill_stats(struct mlx5e_priv *priv, u64 *data,
716
-					int idx)
971
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(eth_ext)
717
972
 {
718
973
 	int i;
719
974
 
..
..
@@ -725,7 +980,7 @@
725
980
 	return idx;
726
981
 }
727
982
 
728
-static void mlx5e_grp_eth_ext_update_stats(struct mlx5e_priv *priv)
983
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(eth_ext)
729
984
 {
730
985
 	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
731
986
 	struct mlx5_core_dev *mdev = priv->mdev;
..
..
@@ -766,7 +1021,7 @@
766
1021
 #define NUM_PCIE_PERF_COUNTERS64	ARRAY_SIZE(pcie_perf_stats_desc64)
767
1022
 #define NUM_PCIE_PERF_STALL_COUNTERS	ARRAY_SIZE(pcie_perf_stall_stats_desc)
768
1023
 
769
-static int mlx5e_grp_pcie_get_num_stats(struct mlx5e_priv *priv)
1024
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(pcie)
770
1025
 {
771
1026
 	int num_stats = 0;
772
1027
 
..
..
@@ -782,8 +1037,7 @@
782
1037
 	return num_stats;
783
1038
 }
784
1039
 
785
-static int mlx5e_grp_pcie_fill_strings(struct mlx5e_priv *priv, u8 *data,
786
-				       int idx)
1040
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(pcie)
787
1041
 {
788
1042
 	int i;
789
1043
 
..
..
@@ -804,8 +1058,7 @@
804
1058
 	return idx;
805
1059
 }
806
1060
 
807
-static int mlx5e_grp_pcie_fill_stats(struct mlx5e_priv *priv, u64 *data,
808
-				     int idx)
1061
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(pcie)
809
1062
 {
810
1063
 	int i;
811
1064
 
..
..
@@ -829,7 +1082,7 @@
829
1082
 	return idx;
830
1083
 }
831
1084
 
832
-static void mlx5e_grp_pcie_update_stats(struct mlx5e_priv *priv)
1085
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(pcie)
833
1086
 {
834
1087
 	struct mlx5e_pcie_stats *pcie_stats = &priv->stats.pcie;
835
1088
 	struct mlx5_core_dev *mdev = priv->mdev;
..
..
@@ -845,12 +1098,152 @@
845
1098
 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
846
1099
 }
847
1100
 
1101
+#define PPORT_PER_TC_PRIO_OFF(c) \
1102
+	MLX5_BYTE_OFF(ppcnt_reg, \
1103
+		      counter_set.eth_per_tc_prio_grp_data_layout.c##_high)
1104
+
1105
+static const struct counter_desc pport_per_tc_prio_stats_desc[] = {
1106
+	{ "rx_prio%d_buf_discard", PPORT_PER_TC_PRIO_OFF(no_buffer_discard_uc) },
1107
+};
1108
+
1109
+#define NUM_PPORT_PER_TC_PRIO_COUNTERS	ARRAY_SIZE(pport_per_tc_prio_stats_desc)
1110
+
1111
+#define PPORT_PER_TC_CONGEST_PRIO_OFF(c) \
1112
+	MLX5_BYTE_OFF(ppcnt_reg, \
1113
+		      counter_set.eth_per_tc_congest_prio_grp_data_layout.c##_high)
1114
+
1115
+static const struct counter_desc pport_per_tc_congest_prio_stats_desc[] = {
1116
+	{ "rx_prio%d_cong_discard", PPORT_PER_TC_CONGEST_PRIO_OFF(wred_discard) },
1117
+	{ "rx_prio%d_marked", PPORT_PER_TC_CONGEST_PRIO_OFF(ecn_marked_tc) },
1118
+};
1119
+
1120
+#define NUM_PPORT_PER_TC_CONGEST_PRIO_COUNTERS \
1121
+	ARRAY_SIZE(pport_per_tc_congest_prio_stats_desc)
1122
+
1123
+static int mlx5e_grp_per_tc_prio_get_num_stats(struct mlx5e_priv *priv)
1124
+{
1125
+	struct mlx5_core_dev *mdev = priv->mdev;
1126
+
1127
+	if (!MLX5_CAP_GEN(mdev, sbcam_reg))
1128
+		return 0;
1129
+
1130
+	return NUM_PPORT_PER_TC_PRIO_COUNTERS * NUM_PPORT_PRIO;
1131
+}
1132
+
1133
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(per_port_buff_congest)
1134
+{
1135
+	struct mlx5_core_dev *mdev = priv->mdev;
1136
+	int i, prio;
1137
+
1138
+	if (!MLX5_CAP_GEN(mdev, sbcam_reg))
1139
+		return idx;
1140
+
1141
+	for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
1142
+		for (i = 0; i < NUM_PPORT_PER_TC_PRIO_COUNTERS; i++)
1143
+			sprintf(data + (idx++) * ETH_GSTRING_LEN,
1144
+				pport_per_tc_prio_stats_desc[i].format, prio);
1145
+		for (i = 0; i < NUM_PPORT_PER_TC_CONGEST_PRIO_COUNTERS; i++)
1146
+			sprintf(data + (idx++) * ETH_GSTRING_LEN,
1147
+				pport_per_tc_congest_prio_stats_desc[i].format, prio);
1148
+	}
1149
+
1150
+	return idx;
1151
+}
1152
+
1153
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(per_port_buff_congest)
1154
+{
1155
+	struct mlx5e_pport_stats *pport = &priv->stats.pport;
1156
+	struct mlx5_core_dev *mdev = priv->mdev;
1157
+	int i, prio;
1158
+
1159
+	if (!MLX5_CAP_GEN(mdev, sbcam_reg))
1160
+		return idx;
1161
+
1162
+	for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
1163
+		for (i = 0; i < NUM_PPORT_PER_TC_PRIO_COUNTERS; i++)
1164
+			data[idx++] =
1165
+				MLX5E_READ_CTR64_BE(&pport->per_tc_prio_counters[prio],
1166
+						    pport_per_tc_prio_stats_desc, i);
1167
+		for (i = 0; i < NUM_PPORT_PER_TC_CONGEST_PRIO_COUNTERS ; i++)
1168
+			data[idx++] =
1169
+				MLX5E_READ_CTR64_BE(&pport->per_tc_congest_prio_counters[prio],
1170
+						    pport_per_tc_congest_prio_stats_desc, i);
1171
+	}
1172
+
1173
+	return idx;
1174
+}
1175
+
1176
+static void mlx5e_grp_per_tc_prio_update_stats(struct mlx5e_priv *priv)
1177
+{
1178
+	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
1179
+	struct mlx5_core_dev *mdev = priv->mdev;
1180
+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
1181
+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
1182
+	void *out;
1183
+	int prio;
1184
+
1185
+	if (!MLX5_CAP_GEN(mdev, sbcam_reg))
1186
+		return;
1187
+
1188
+	MLX5_SET(ppcnt_reg, in, pnat, 2);
1189
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_TRAFFIC_CLASS_COUNTERS_GROUP);
1190
+	for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
1191
+		out = pstats->per_tc_prio_counters[prio];
1192
+		MLX5_SET(ppcnt_reg, in, prio_tc, prio);
1193
+		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
1194
+	}
1195
+}
1196
+
1197
+static int mlx5e_grp_per_tc_congest_prio_get_num_stats(struct mlx5e_priv *priv)
1198
+{
1199
+	struct mlx5_core_dev *mdev = priv->mdev;
1200
+
1201
+	if (!MLX5_CAP_GEN(mdev, sbcam_reg))
1202
+		return 0;
1203
+
1204
+	return NUM_PPORT_PER_TC_CONGEST_PRIO_COUNTERS * NUM_PPORT_PRIO;
1205
+}
1206
+
1207
+static void mlx5e_grp_per_tc_congest_prio_update_stats(struct mlx5e_priv *priv)
1208
+{
1209
+	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
1210
+	struct mlx5_core_dev *mdev = priv->mdev;
1211
+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
1212
+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
1213
+	void *out;
1214
+	int prio;
1215
+
1216
+	if (!MLX5_CAP_GEN(mdev, sbcam_reg))
1217
+		return;
1218
+
1219
+	MLX5_SET(ppcnt_reg, in, pnat, 2);
1220
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_TRAFFIC_CLASS_CONGESTION_GROUP);
1221
+	for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
1222
+		out = pstats->per_tc_congest_prio_counters[prio];
1223
+		MLX5_SET(ppcnt_reg, in, prio_tc, prio);
1224
+		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
1225
+	}
1226
+}
1227
+
1228
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(per_port_buff_congest)
1229
+{
1230
+	return mlx5e_grp_per_tc_prio_get_num_stats(priv) +
1231
+		mlx5e_grp_per_tc_congest_prio_get_num_stats(priv);
1232
+}
1233
+
1234
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(per_port_buff_congest)
1235
+{
1236
+	mlx5e_grp_per_tc_prio_update_stats(priv);
1237
+	mlx5e_grp_per_tc_congest_prio_update_stats(priv);
1238
+}
1239
+
848
1240
 #define PPORT_PER_PRIO_OFF(c) \
849
1241
 	MLX5_BYTE_OFF(ppcnt_reg, \
850
1242
 		      counter_set.eth_per_prio_grp_data_layout.c##_high)
851
1243
 static const struct counter_desc pport_per_prio_traffic_stats_desc[] = {
852
1244
 	{ "rx_prio%d_bytes", PPORT_PER_PRIO_OFF(rx_octets) },
853
1245
 	{ "rx_prio%d_packets", PPORT_PER_PRIO_OFF(rx_frames) },
1246
+	{ "rx_prio%d_discards", PPORT_PER_PRIO_OFF(rx_discards) },
854
1247
 	{ "tx_prio%d_bytes", PPORT_PER_PRIO_OFF(tx_octets) },
855
1248
 	{ "tx_prio%d_packets", PPORT_PER_PRIO_OFF(tx_frames) },
856
1249
 };
..
..
@@ -903,7 +1296,7 @@
903
1296
 };
904
1297
 
905
1298
 static const struct counter_desc pport_pfc_stall_stats_desc[] = {
906
-	{ "tx_pause_storm_warning_events ", PPORT_PER_PRIO_OFF(device_stall_minor_watermark_cnt) },
1299
+	{ "tx_pause_storm_warning_events", PPORT_PER_PRIO_OFF(device_stall_minor_watermark_cnt) },
907
1300
 	{ "tx_pause_storm_error_events", PPORT_PER_PRIO_OFF(device_stall_critical_watermark_cnt) },
908
1301
 };
909
1302
 
..
..
@@ -1013,29 +1406,27 @@
1013
1406
 	return idx;
1014
1407
 }
1015
1408
 
1016
-static int mlx5e_grp_per_prio_get_num_stats(struct mlx5e_priv *priv)
1409
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(per_prio)
1017
1410
 {
1018
1411
 	return mlx5e_grp_per_prio_traffic_get_num_stats() +
1019
1412
 		mlx5e_grp_per_prio_pfc_get_num_stats(priv);
1020
1413
 }
1021
1414
 
1022
-static int mlx5e_grp_per_prio_fill_strings(struct mlx5e_priv *priv, u8 *data,
1023
-					   int idx)
1415
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(per_prio)
1024
1416
 {
1025
1417
 	idx = mlx5e_grp_per_prio_traffic_fill_strings(priv, data, idx);
1026
1418
 	idx = mlx5e_grp_per_prio_pfc_fill_strings(priv, data, idx);
1027
1419
 	return idx;
1028
1420
 }
1029
1421
 
1030
-static int mlx5e_grp_per_prio_fill_stats(struct mlx5e_priv *priv, u64 *data,
1031
-					 int idx)
1422
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(per_prio)
1032
1423
 {
1033
1424
 	idx = mlx5e_grp_per_prio_traffic_fill_stats(priv, data, idx);
1034
1425
 	idx = mlx5e_grp_per_prio_pfc_fill_stats(priv, data, idx);
1035
1426
 	return idx;
1036
1427
 }
1037
1428
 
1038
-static void mlx5e_grp_per_prio_update_stats(struct mlx5e_priv *priv)
1429
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(per_prio)
1039
1430
 {
1040
1431
 	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
1041
1432
 	struct mlx5_core_dev *mdev = priv->mdev;
..
..
@@ -1043,6 +1434,9 @@
1043
1434
 	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
1044
1435
 	int prio;
1045
1436
 	void *out;
1437
+
1438
+	if (!MLX5_BASIC_PPCNT_SUPPORTED(mdev))
1439
+		return;
1046
1440
 
1047
1441
 	MLX5_SET(ppcnt_reg, in, local_port, 1);
1048
1442
 	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
..
..
@@ -1055,25 +1449,24 @@
1055
1449
 }
1056
1450
 
1057
1451
 static const struct counter_desc mlx5e_pme_status_desc[] = {
1058
-	{ "module_unplug", 8 },
1452
+	{ "module_unplug",       sizeof(u64) * MLX5_MODULE_STATUS_UNPLUGGED },
1059
1453
 };
1060
1454
 
1061
1455
 static const struct counter_desc mlx5e_pme_error_desc[] = {
1062
-	{ "module_bus_stuck", 16 },       /* bus stuck (I2C or data shorted) */
1063
-	{ "module_high_temp", 48 },       /* high temperature */
1064
-	{ "module_bad_shorted", 56 },    /* bad or shorted cable/module */
1456
+	{ "module_bus_stuck",    sizeof(u64) * MLX5_MODULE_EVENT_ERROR_BUS_STUCK },
1457
+	{ "module_high_temp",    sizeof(u64) * MLX5_MODULE_EVENT_ERROR_HIGH_TEMPERATURE },
1458
+	{ "module_bad_shorted",  sizeof(u64) * MLX5_MODULE_EVENT_ERROR_BAD_CABLE },
1065
1459
 };
1066
1460
 
1067
1461
 #define NUM_PME_STATUS_STATS		ARRAY_SIZE(mlx5e_pme_status_desc)
1068
1462
 #define NUM_PME_ERR_STATS		ARRAY_SIZE(mlx5e_pme_error_desc)
1069
1463
 
1070
-static int mlx5e_grp_pme_get_num_stats(struct mlx5e_priv *priv)
1464
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(pme)
1071
1465
 {
1072
1466
 	return NUM_PME_STATUS_STATS + NUM_PME_ERR_STATS;
1073
1467
 }
1074
1468
 
1075
-static int mlx5e_grp_pme_fill_strings(struct mlx5e_priv *priv, u8 *data,
1076
-				      int idx)
1469
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(pme)
1077
1470
 {
1078
1471
 	int i;
1079
1472
 
..
..
@@ -1086,61 +1479,42 @@
1086
1479
 	return idx;
1087
1480
 }
1088
1481
 
1089
-static int mlx5e_grp_pme_fill_stats(struct mlx5e_priv *priv, u64 *data,
1090
-				    int idx)
1482
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(pme)
1091
1483
 {
1092
-	struct mlx5_priv *mlx5_priv = &priv->mdev->priv;
1484
+	struct mlx5_pme_stats pme_stats;
1093
1485
 	int i;
1094
1486
 
1487
+	mlx5_get_pme_stats(priv->mdev, &pme_stats);
1488
+
1095
1489
 	for (i = 0; i < NUM_PME_STATUS_STATS; i++)
1096
-		data[idx++] = MLX5E_READ_CTR64_CPU(mlx5_priv->pme_stats.status_counters,
1490
+		data[idx++] = MLX5E_READ_CTR64_CPU(pme_stats.status_counters,
1097
1491
 						   mlx5e_pme_status_desc, i);
1098
1492
 
1099
1493
 	for (i = 0; i < NUM_PME_ERR_STATS; i++)
1100
-		data[idx++] = MLX5E_READ_CTR64_CPU(mlx5_priv->pme_stats.error_counters,
1494
+		data[idx++] = MLX5E_READ_CTR64_CPU(pme_stats.error_counters,
1101
1495
 						   mlx5e_pme_error_desc, i);
1102
1496
 
1103
1497
 	return idx;
1104
1498
 }
1105
1499
 
1106
-static int mlx5e_grp_ipsec_get_num_stats(struct mlx5e_priv *priv)
1107
-{
1108
-	return mlx5e_ipsec_get_count(priv);
1109
-}
1500
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(pme) { return; }
1110
1501
 
1111
-static int mlx5e_grp_ipsec_fill_strings(struct mlx5e_priv *priv, u8 *data,
1112
-					int idx)
1113
-{
1114
-	return idx + mlx5e_ipsec_get_strings(priv,
1115
-					     data + idx * ETH_GSTRING_LEN);
1116
-}
1117
-
1118
-static int mlx5e_grp_ipsec_fill_stats(struct mlx5e_priv *priv, u64 *data,
1119
-				      int idx)
1120
-{
1121
-	return idx + mlx5e_ipsec_get_stats(priv, data + idx);
1122
-}
1123
-
1124
-static void mlx5e_grp_ipsec_update_stats(struct mlx5e_priv *priv)
1125
-{
1126
-	mlx5e_ipsec_update_stats(priv);
1127
-}
1128
-
1129
-static int mlx5e_grp_tls_get_num_stats(struct mlx5e_priv *priv)
1502
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(tls)
1130
1503
 {
1131
1504
 	return mlx5e_tls_get_count(priv);
1132
1505
 }
1133
1506
 
1134
-static int mlx5e_grp_tls_fill_strings(struct mlx5e_priv *priv, u8 *data,
1135
-				      int idx)
1507
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(tls)
1136
1508
 {
1137
1509
 	return idx + mlx5e_tls_get_strings(priv, data + idx * ETH_GSTRING_LEN);
1138
1510
 }
1139
1511
 
1140
-static int mlx5e_grp_tls_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
1512
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(tls)
1141
1513
 {
1142
1514
 	return idx + mlx5e_tls_get_stats(priv, data + idx);
1143
1515
 }
1516
+
1517
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(tls) { return; }
1144
1518
 
1145
1519
 static const struct counter_desc rq_stats_desc[] = {
1146
1520
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, packets) },
..
..
@@ -1164,13 +1538,27 @@
1164
1538
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, buff_alloc_err) },
1165
1539
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_blks) },
1166
1540
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) },
1167
-	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, page_reuse) },
1168
1541
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_reuse) },
1169
1542
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) },
1170
1543
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) },
1171
1544
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_busy) },
1172
1545
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_waive) },
1173
1546
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, congst_umr) },
1547
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, arfs_err) },
1548
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, recover) },
1549
+#ifdef CONFIG_MLX5_EN_TLS
1550
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_decrypted_packets) },
1551
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_decrypted_bytes) },
1552
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_ctx) },
1553
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_del) },
1554
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_pkt) },
1555
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_start) },
1556
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_end) },
1557
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_req_skip) },
1558
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_res_ok) },
1559
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_resync_res_skip) },
1560
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, tls_err) },
1561
+#endif
1174
1562
 };
1175
1563
 
1176
1564
 static const struct counter_desc sq_stats_desc[] = {
..
..
@@ -1184,6 +1572,20 @@
1184
1572
 	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, csum_partial_inner) },
1185
1573
 	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, added_vlan_packets) },
1186
1574
 	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, nop) },
1575
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, mpwqe_blks) },
1576
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, mpwqe_pkts) },
1577
+#ifdef CONFIG_MLX5_EN_TLS
1578
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_encrypted_packets) },
1579
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_encrypted_bytes) },
1580
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_ctx) },
1581
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_ooo) },
1582
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_dump_packets) },
1583
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_dump_bytes) },
1584
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_resync_bytes) },
1585
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_skip_no_sync_data) },
1586
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_drop_no_sync_data) },
1587
+	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, tls_drop_bypass_req) },
1588
+#endif
1187
1589
 	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, csum_none) },
1188
1590
 	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, stopped) },
1189
1591
 	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, dropped) },
..
..
@@ -1196,6 +1598,9 @@
1196
1598
 
1197
1599
 static const struct counter_desc rq_xdpsq_stats_desc[] = {
1198
1600
 	{ MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
1601
+	{ MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) },
1602
+	{ MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) },
1603
+	{ MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, nops) },
1199
1604
 	{ MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },
1200
1605
 	{ MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },
1201
1606
 	{ MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
..
..
@@ -1203,9 +1608,43 @@
1203
1608
 
1204
1609
 static const struct counter_desc xdpsq_stats_desc[] = {
1205
1610
 	{ MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
1611
+	{ MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) },
1612
+	{ MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) },
1613
+	{ MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, nops) },
1206
1614
 	{ MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },
1207
1615
 	{ MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },
1208
1616
 	{ MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
1617
+};
1618
+
1619
+static const struct counter_desc xskrq_stats_desc[] = {
1620
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, packets) },
1621
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, bytes) },
1622
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, csum_complete) },
1623
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, csum_unnecessary) },
1624
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, csum_unnecessary_inner) },
1625
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, csum_none) },
1626
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, ecn_mark) },
1627
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, removed_vlan_packets) },
1628
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, xdp_drop) },
1629
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, xdp_redirect) },
1630
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, wqe_err) },
1631
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, mpwqe_filler_cqes) },
1632
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, mpwqe_filler_strides) },
1633
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, oversize_pkts_sw_drop) },
1634
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, buff_alloc_err) },
1635
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, cqe_compress_blks) },
1636
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) },
1637
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, congst_umr) },
1638
+	{ MLX5E_DECLARE_XSKRQ_STAT(struct mlx5e_rq_stats, arfs_err) },
1639
+};
1640
+
1641
+static const struct counter_desc xsksq_stats_desc[] = {
1642
+	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
1643
+	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) },
1644
+	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) },
1645
+	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, full) },
1646
+	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, err) },
1647
+	{ MLX5E_DECLARE_XSKSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
1209
1648
 };
1210
1649
 
1211
1650
 static const struct counter_desc ch_stats_desc[] = {
..
..
@@ -1213,6 +1652,7 @@
1213
1652
 	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, poll) },
1214
1653
 	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, arm) },
1215
1654
 	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, aff_change) },
1655
+	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, force_irq) },
1216
1656
 	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, eq_rearm) },
1217
1657
 };
1218
1658
 
..
..
@@ -1220,23 +1660,27 @@
1220
1660
 #define NUM_SQ_STATS			ARRAY_SIZE(sq_stats_desc)
1221
1661
 #define NUM_XDPSQ_STATS			ARRAY_SIZE(xdpsq_stats_desc)
1222
1662
 #define NUM_RQ_XDPSQ_STATS		ARRAY_SIZE(rq_xdpsq_stats_desc)
1663
+#define NUM_XSKRQ_STATS			ARRAY_SIZE(xskrq_stats_desc)
1664
+#define NUM_XSKSQ_STATS			ARRAY_SIZE(xsksq_stats_desc)
1223
1665
 #define NUM_CH_STATS			ARRAY_SIZE(ch_stats_desc)
1224
1666
 
1225
-static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv)
1667
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(channels)
1226
1668
 {
1227
-	int max_nch = priv->profile->max_nch(priv->mdev);
1669
+	int max_nch = priv->max_nch;
1228
1670
 
1229
1671
 	return (NUM_RQ_STATS * max_nch) +
1230
1672
 	       (NUM_CH_STATS * max_nch) +
1231
1673
 	       (NUM_SQ_STATS * max_nch * priv->max_opened_tc) +
1232
1674
 	       (NUM_RQ_XDPSQ_STATS * max_nch) +
1233
-	       (NUM_XDPSQ_STATS * max_nch);
1675
+	       (NUM_XDPSQ_STATS * max_nch) +
1676
+	       (NUM_XSKRQ_STATS * max_nch * priv->xsk.ever_used) +
1677
+	       (NUM_XSKSQ_STATS * max_nch * priv->xsk.ever_used);
1234
1678
 }
1235
1679
 
1236
-static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv *priv, u8 *data,
1237
-					   int idx)
1680
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(channels)
1238
1681
 {
1239
-	int max_nch = priv->profile->max_nch(priv->mdev);
1682
+	bool is_xsk = priv->xsk.ever_used;
1683
+	int max_nch = priv->max_nch;
1240
1684
 	int i, j, tc;
1241
1685
 
1242
1686
 	for (i = 0; i < max_nch; i++)
..
..
@@ -1248,6 +1692,9 @@
1248
1692
 		for (j = 0; j < NUM_RQ_STATS; j++)
1249
1693
 			sprintf(data + (idx++) * ETH_GSTRING_LEN,
1250
1694
 				rq_stats_desc[j].format, i);
1695
+		for (j = 0; j < NUM_XSKRQ_STATS * is_xsk; j++)
1696
+			sprintf(data + (idx++) * ETH_GSTRING_LEN,
1697
+				xskrq_stats_desc[j].format, i);
1251
1698
 		for (j = 0; j < NUM_RQ_XDPSQ_STATS; j++)
1252
1699
 			sprintf(data + (idx++) * ETH_GSTRING_LEN,
1253
1700
 				rq_xdpsq_stats_desc[j].format, i);
..
..
@@ -1258,20 +1705,24 @@
1258
1705
 			for (j = 0; j < NUM_SQ_STATS; j++)
1259
1706
 				sprintf(data + (idx++) * ETH_GSTRING_LEN,
1260
1707
 					sq_stats_desc[j].format,
1261
-					priv->channel_tc2txq[i][tc]);
1708
+					i + tc * max_nch);
1262
1709
 
1263
-	for (i = 0; i < max_nch; i++)
1710
+	for (i = 0; i < max_nch; i++) {
1711
+		for (j = 0; j < NUM_XSKSQ_STATS * is_xsk; j++)
1712
+			sprintf(data + (idx++) * ETH_GSTRING_LEN,
1713
+				xsksq_stats_desc[j].format, i);
1264
1714
 		for (j = 0; j < NUM_XDPSQ_STATS; j++)
1265
1715
 			sprintf(data + (idx++) * ETH_GSTRING_LEN,
1266
1716
 				xdpsq_stats_desc[j].format, i);
1717
+	}
1267
1718
 
1268
1719
 	return idx;
1269
1720
 }
1270
1721
 
1271
-static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv *priv, u64 *data,
1272
-					 int idx)
1722
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(channels)
1273
1723
 {
1274
-	int max_nch = priv->profile->max_nch(priv->mdev);
1724
+	bool is_xsk = priv->xsk.ever_used;
1725
+	int max_nch = priv->max_nch;
1275
1726
 	int i, j, tc;
1276
1727
 
1277
1728
 	for (i = 0; i < max_nch; i++)
..
..
@@ -1285,6 +1736,10 @@
1285
1736
 			data[idx++] =
1286
1737
 				MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq,
1287
1738
 						     rq_stats_desc, j);
1739
+		for (j = 0; j < NUM_XSKRQ_STATS * is_xsk; j++)
1740
+			data[idx++] =
1741
+				MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].xskrq,
1742
+						     xskrq_stats_desc, j);
1288
1743
 		for (j = 0; j < NUM_RQ_XDPSQ_STATS; j++)
1289
1744
 			data[idx++] =
1290
1745
 				MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq_xdpsq,
..
..
@@ -1298,107 +1753,62 @@
1298
1753
 					MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].sq[tc],
1299
1754
 							     sq_stats_desc, j);
1300
1755
 
1301
-	for (i = 0; i < max_nch; i++)
1756
+	for (i = 0; i < max_nch; i++) {
1757
+		for (j = 0; j < NUM_XSKSQ_STATS * is_xsk; j++)
1758
+			data[idx++] =
1759
+				MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].xsksq,
1760
+						     xsksq_stats_desc, j);
1302
1761
 		for (j = 0; j < NUM_XDPSQ_STATS; j++)
1303
1762
 			data[idx++] =
1304
1763
 				MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].xdpsq,
1305
1764
 						     xdpsq_stats_desc, j);
1765
+	}
1306
1766
 
1307
1767
 	return idx;
1308
1768
 }
1309
1769
 
1770
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(channels) { return; }
1771
+
1772
+MLX5E_DEFINE_STATS_GRP(sw, 0);
1773
+MLX5E_DEFINE_STATS_GRP(qcnt, MLX5E_NDO_UPDATE_STATS);
1774
+MLX5E_DEFINE_STATS_GRP(vnic_env, 0);
1775
+MLX5E_DEFINE_STATS_GRP(vport, MLX5E_NDO_UPDATE_STATS);
1776
+MLX5E_DEFINE_STATS_GRP(802_3, MLX5E_NDO_UPDATE_STATS);
1777
+MLX5E_DEFINE_STATS_GRP(2863, 0);
1778
+MLX5E_DEFINE_STATS_GRP(2819, 0);
1779
+MLX5E_DEFINE_STATS_GRP(phy, 0);
1780
+MLX5E_DEFINE_STATS_GRP(pcie, 0);
1781
+MLX5E_DEFINE_STATS_GRP(per_prio, 0);
1782
+MLX5E_DEFINE_STATS_GRP(pme, 0);
1783
+MLX5E_DEFINE_STATS_GRP(channels, 0);
1784
+MLX5E_DEFINE_STATS_GRP(per_port_buff_congest, 0);
1785
+MLX5E_DEFINE_STATS_GRP(eth_ext, 0);
1786
+static MLX5E_DEFINE_STATS_GRP(tls, 0);
1787
+
1310
1788
 /* The stats groups order is opposite to the update_stats() order calls */
1311
-const struct mlx5e_stats_grp mlx5e_stats_grps[] = {
1312
-	{
1313
-		.get_num_stats = mlx5e_grp_sw_get_num_stats,
1314
-		.fill_strings = mlx5e_grp_sw_fill_strings,
1315
-		.fill_stats = mlx5e_grp_sw_fill_stats,
1316
-		.update_stats = mlx5e_grp_sw_update_stats,
1317
-	},
1318
-	{
1319
-		.get_num_stats = mlx5e_grp_q_get_num_stats,
1320
-		.fill_strings = mlx5e_grp_q_fill_strings,
1321
-		.fill_stats = mlx5e_grp_q_fill_stats,
1322
-		.update_stats_mask = MLX5E_NDO_UPDATE_STATS,
1323
-		.update_stats = mlx5e_grp_q_update_stats,
1324
-	},
1325
-	{
1326
-		.get_num_stats = mlx5e_grp_vnic_env_get_num_stats,
1327
-		.fill_strings = mlx5e_grp_vnic_env_fill_strings,
1328
-		.fill_stats = mlx5e_grp_vnic_env_fill_stats,
1329
-		.update_stats = mlx5e_grp_vnic_env_update_stats,
1330
-	},
1331
-	{
1332
-		.get_num_stats = mlx5e_grp_vport_get_num_stats,
1333
-		.fill_strings = mlx5e_grp_vport_fill_strings,
1334
-		.fill_stats = mlx5e_grp_vport_fill_stats,
1335
-		.update_stats_mask = MLX5E_NDO_UPDATE_STATS,
1336
-		.update_stats = mlx5e_grp_vport_update_stats,
1337
-	},
1338
-	{
1339
-		.get_num_stats = mlx5e_grp_802_3_get_num_stats,
1340
-		.fill_strings = mlx5e_grp_802_3_fill_strings,
1341
-		.fill_stats = mlx5e_grp_802_3_fill_stats,
1342
-		.update_stats_mask = MLX5E_NDO_UPDATE_STATS,
1343
-		.update_stats = mlx5e_grp_802_3_update_stats,
1344
-	},
1345
-	{
1346
-		.get_num_stats = mlx5e_grp_2863_get_num_stats,
1347
-		.fill_strings = mlx5e_grp_2863_fill_strings,
1348
-		.fill_stats = mlx5e_grp_2863_fill_stats,
1349
-		.update_stats = mlx5e_grp_2863_update_stats,
1350
-	},
1351
-	{
1352
-		.get_num_stats = mlx5e_grp_2819_get_num_stats,
1353
-		.fill_strings = mlx5e_grp_2819_fill_strings,
1354
-		.fill_stats = mlx5e_grp_2819_fill_stats,
1355
-		.update_stats = mlx5e_grp_2819_update_stats,
1356
-	},
1357
-	{
1358
-		.get_num_stats = mlx5e_grp_phy_get_num_stats,
1359
-		.fill_strings = mlx5e_grp_phy_fill_strings,
1360
-		.fill_stats = mlx5e_grp_phy_fill_stats,
1361
-		.update_stats = mlx5e_grp_phy_update_stats,
1362
-	},
1363
-	{
1364
-		.get_num_stats = mlx5e_grp_eth_ext_get_num_stats,
1365
-		.fill_strings = mlx5e_grp_eth_ext_fill_strings,
1366
-		.fill_stats = mlx5e_grp_eth_ext_fill_stats,
1367
-		.update_stats = mlx5e_grp_eth_ext_update_stats,
1368
-	},
1369
-	{
1370
-		.get_num_stats = mlx5e_grp_pcie_get_num_stats,
1371
-		.fill_strings = mlx5e_grp_pcie_fill_strings,
1372
-		.fill_stats = mlx5e_grp_pcie_fill_stats,
1373
-		.update_stats = mlx5e_grp_pcie_update_stats,
1374
-	},
1375
-	{
1376
-		.get_num_stats = mlx5e_grp_per_prio_get_num_stats,
1377
-		.fill_strings = mlx5e_grp_per_prio_fill_strings,
1378
-		.fill_stats = mlx5e_grp_per_prio_fill_stats,
1379
-		.update_stats = mlx5e_grp_per_prio_update_stats,
1380
-	},
1381
-	{
1382
-		.get_num_stats = mlx5e_grp_pme_get_num_stats,
1383
-		.fill_strings = mlx5e_grp_pme_fill_strings,
1384
-		.fill_stats = mlx5e_grp_pme_fill_stats,
1385
-	},
1386
-	{
1387
-		.get_num_stats = mlx5e_grp_ipsec_get_num_stats,
1388
-		.fill_strings = mlx5e_grp_ipsec_fill_strings,
1389
-		.fill_stats = mlx5e_grp_ipsec_fill_stats,
1390
-		.update_stats = mlx5e_grp_ipsec_update_stats,
1391
-	},
1392
-	{
1393
-		.get_num_stats = mlx5e_grp_tls_get_num_stats,
1394
-		.fill_strings = mlx5e_grp_tls_fill_strings,
1395
-		.fill_stats = mlx5e_grp_tls_fill_stats,
1396
-	},
1397
-	{
1398
-		.get_num_stats = mlx5e_grp_channels_get_num_stats,
1399
-		.fill_strings = mlx5e_grp_channels_fill_strings,
1400
-		.fill_stats = mlx5e_grp_channels_fill_stats,
1401
-	}
1789
+mlx5e_stats_grp_t mlx5e_nic_stats_grps[] = {
1790
+	&MLX5E_STATS_GRP(sw),
1791
+	&MLX5E_STATS_GRP(qcnt),
1792
+	&MLX5E_STATS_GRP(vnic_env),
1793
+	&MLX5E_STATS_GRP(vport),
1794
+	&MLX5E_STATS_GRP(802_3),
1795
+	&MLX5E_STATS_GRP(2863),
1796
+	&MLX5E_STATS_GRP(2819),
1797
+	&MLX5E_STATS_GRP(phy),
1798
+	&MLX5E_STATS_GRP(eth_ext),
1799
+	&MLX5E_STATS_GRP(pcie),
1800
+	&MLX5E_STATS_GRP(per_prio),
1801
+	&MLX5E_STATS_GRP(pme),
1802
+#ifdef CONFIG_MLX5_EN_IPSEC
1803
+	&MLX5E_STATS_GRP(ipsec_sw),
1804
+	&MLX5E_STATS_GRP(ipsec_hw),
1805
+#endif
1806
+	&MLX5E_STATS_GRP(tls),
1807
+	&MLX5E_STATS_GRP(channels),
1808
+	&MLX5E_STATS_GRP(per_port_buff_congest),
1402
1809
 };
1403
1810
 
1404
-const int mlx5e_num_stats_grps = ARRAY_SIZE(mlx5e_stats_grps);
1811
+unsigned int mlx5e_nic_stats_grps_num(struct mlx5e_priv *priv)
1812
+{
1813
+	return ARRAY_SIZE(mlx5e_nic_stats_grps);
1814
+}