forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-04 1543e317f1da31b75942316931e8f491a8920811 
 kernel/drivers/net/ethernet/amazon/ena/ena_netdev.h
....@@ -1,40 +1,15 @@
1
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
12 /*
2
- * Copyright 2015 Amazon.com, Inc. or its affiliates.
3
- *
4
- * This software is available to you under a choice of one of two
5
- * licenses.  You may choose to be licensed under the terms of the GNU
6
- * General Public License (GPL) Version 2, available from the file
7
- * COPYING in the main directory of this source tree, or the
8
- * BSD license below:
9
- *
10
- *     Redistribution and use in source and binary forms, with or
11
- *     without modification, are permitted provided that the following
12
- *     conditions are met:
13
- *
14
- *      - Redistributions of source code must retain the above
15
- *        copyright notice, this list of conditions and the following
16
- *        disclaimer.
17
- *
18
- *      - Redistributions in binary form must reproduce the above
19
- *        copyright notice, this list of conditions and the following
20
- *        disclaimer in the documentation and/or other materials
21
- *        provided with the distribution.
22
- *
23
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30
- * SOFTWARE.
3
+ * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
314  */
325 
336 #ifndef ENA_H
347 #define ENA_H
358 
369 #include <linux/bitops.h>
10
+#include <linux/dim.h>
3711 #include <linux/etherdevice.h>
12
+#include <linux/if_vlan.h>
3813 #include <linux/inetdevice.h>
3914 #include <linux/interrupt.h>
4015 #include <linux/netdevice.h>
....@@ -43,23 +18,28 @@
4318 #include "ena_com.h"
4419 #include "ena_eth_com.h"
4520 
46
-#define DRV_MODULE_VER_MAJOR	1
47
-#define DRV_MODULE_VER_MINOR	5
48
-#define DRV_MODULE_VER_SUBMINOR 0
21
+#define DRV_MODULE_GEN_MAJOR	2
22
+#define DRV_MODULE_GEN_MINOR	1
23
+#define DRV_MODULE_GEN_SUBMINOR 0
4924 
5025 #define DRV_MODULE_NAME		"ena"
51
-#ifndef DRV_MODULE_VERSION
52
-#define DRV_MODULE_VERSION \
53
-	__stringify(DRV_MODULE_VER_MAJOR) "."	\
54
-	__stringify(DRV_MODULE_VER_MINOR) "."	\
55
-	__stringify(DRV_MODULE_VER_SUBMINOR) "K"
56
-#endif
5726 
5827 #define DEVICE_NAME	"Elastic Network Adapter (ENA)"
5928 
6029 /* 1 for AENQ + ADMIN */
6130 #define ENA_ADMIN_MSIX_VEC		1
6231 #define ENA_MAX_MSIX_VEC(io_queues)	(ENA_ADMIN_MSIX_VEC + (io_queues))
32
+
33
+/* The ENA buffer length fields is 16 bit long. So when PAGE_SIZE == 64kB the
34
+ * driver passes 0.
35
+ * Since the max packet size the ENA handles is ~9kB limit the buffer length to
36
+ * 16kB.
37
+ */
38
+#if PAGE_SIZE > SZ_16K
39
+#define ENA_PAGE_SIZE (_AC(SZ_16K, UL))
40
+#else
41
+#define ENA_PAGE_SIZE PAGE_SIZE
42
+#endif
6343 
6444 #define ENA_MIN_MSIX_VEC		2
6545 
....@@ -68,9 +48,12 @@
6848 #define ENA_BAR_MASK (BIT(ENA_REG_BAR) | BIT(ENA_MEM_BAR))
6949 
7050 #define ENA_DEFAULT_RING_SIZE	(1024)
51
+#define ENA_MIN_RING_SIZE	(256)
52
+
53
+#define ENA_MIN_NUM_IO_QUEUES	(1)
7154 
7255 #define ENA_TX_WAKEUP_THRESH		(MAX_SKB_FRAGS + 2)
73
-#define ENA_DEFAULT_RX_COPYBREAK	(128 - NET_IP_ALIGN)
56
+#define ENA_DEFAULT_RX_COPYBREAK	(256 - NET_IP_ALIGN)
7457 
7558 /* limit the buffer size to 600 bytes to handle MTU changes from very
7659  * small to very large, in which case the number of buffers per packet
....@@ -88,17 +71,16 @@
8871 #define ENA_RX_RSS_TABLE_LOG_SIZE  7
8972 #define ENA_RX_RSS_TABLE_SIZE	(1 << ENA_RX_RSS_TABLE_LOG_SIZE)
9073 
91
-#define ENA_HASH_KEY_SIZE	40
92
-
9374 /* The number of tx packet completions that will be handled each NAPI poll
9475  * cycle is ring_size / ENA_TX_POLL_BUDGET_DIVIDER.
9576  */
9677 #define ENA_TX_POLL_BUDGET_DIVIDER	4
9778 
98
-/* Refill Rx queue when number of available descriptors is below
99
- * QUEUE_SIZE / ENA_RX_REFILL_THRESH_DIVIDER
79
+/* Refill Rx queue when number of required descriptors is above
80
+ * QUEUE_SIZE / ENA_RX_REFILL_THRESH_DIVIDER or ENA_RX_REFILL_THRESH_PACKET
10081  */
10182 #define ENA_RX_REFILL_THRESH_DIVIDER	8
83
+#define ENA_RX_REFILL_THRESH_PACKET	256
10284 
10385 /* Number of queues to check for missing queues per timer service */
10486 #define ENA_MONITORED_TX_QUEUES	4
....@@ -120,6 +102,8 @@
120102 #define ENA_IO_IRQ_FIRST_IDX		1
121103 #define ENA_IO_IRQ_IDX(q)		(ENA_IO_IRQ_FIRST_IDX + (q))
122104 
105
+#define ENA_ADMIN_POLL_DELAY_US 100
106
+
123107 /* ENA device should send keep alive msg every 1 sec.
124108  * We wait for 6 sec just to be on the safe side.
125109  */
....@@ -127,6 +111,19 @@
127111 #define ENA_MAX_NO_INTERRUPT_ITERATIONS 3
128112 
129113 #define ENA_MMIO_DISABLE_REG_READ	BIT(0)
114
+
115
+/* The max MTU size is configured to be the ethernet frame size without
116
+ * the overhead of the ethernet header, which can have a VLAN header, and
117
+ * a frame check sequence (FCS).
118
+ * The buffer size we share with the device is defined to be ENA_PAGE_SIZE
119
+ */
120
+
121
+#define ENA_XDP_MAX_MTU (ENA_PAGE_SIZE - ETH_HLEN - ETH_FCS_LEN -	\
122
+			 VLAN_HLEN - XDP_PACKET_HEADROOM -		\
123
+			 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
124
+
125
+#define ENA_IS_XDP_INDEX(adapter, index) (((index) >= (adapter)->xdp_first_ring) && \
126
+	((index) < (adapter)->xdp_first_ring + (adapter)->xdp_num_queues))
130127 
131128 struct ena_irq {
132129 	irq_handler_t handler;
....@@ -141,7 +138,23 @@
141138 	struct napi_struct napi ____cacheline_aligned;
142139 	struct ena_ring *tx_ring;
143140 	struct ena_ring *rx_ring;
141
+	struct ena_ring *xdp_ring;
142
+	bool first_interrupt;
143
+	bool interrupts_masked;
144144 	u32 qid;
145
+	struct dim dim;
146
+};
147
+
148
+struct ena_calc_queue_size_ctx {
149
+	struct ena_com_dev_get_features_ctx *get_feat_ctx;
150
+	struct ena_com_dev *ena_dev;
151
+	struct pci_dev *pdev;
152
+	u32 tx_queue_size;
153
+	u32 rx_queue_size;
154
+	u32 max_tx_queue_size;
155
+	u32 max_rx_queue_size;
156
+	u16 max_tx_sgl_size;
157
+	u16 max_rx_sgl_size;
145158 };
146159 
147160 struct ena_tx_buffer {
....@@ -152,6 +165,20 @@
152165 	u32 tx_descs;
153166 	/* num of buffers used by this skb */
154167 	u32 num_of_bufs;
168
+
169
+	/* XDP buffer structure which is used for sending packets in
170
+	 * the xdp queues
171
+	 */
172
+	struct xdp_frame *xdpf;
173
+	/* The rx page for the rx buffer that was received in rx and
174
+	 * re transmitted on xdp tx queues as a result of XDP_TX action.
175
+	 * We need to free the page once we finished cleaning the buffer in
176
+	 * clean_xdp_irq()
177
+	 */
178
+	struct page *xdp_rx_page;
179
+
180
+	/* Indicate if bufs[0] map the linear data of the skb. */
181
+	u8 map_linear_data;
155182 
156183 	/* Used for detect missing tx packets to limit the number of prints */
157184 	u32 print_once;
....@@ -188,31 +215,37 @@
188215 	u64 tx_poll;
189216 	u64 doorbells;
190217 	u64 bad_req_id;
218
+	u64 llq_buffer_copy;
191219 	u64 missed_tx;
220
+	u64 unmask_interrupt;
192221 };
193222 
194223 struct ena_stats_rx {
195224 	u64 cnt;
196225 	u64 bytes;
226
+	u64 rx_copybreak_pkt;
227
+	u64 csum_good;
197228 	u64 refil_partial;
198229 	u64 bad_csum;
199230 	u64 page_alloc_fail;
200231 	u64 skb_alloc_fail;
201232 	u64 dma_mapping_err;
202233 	u64 bad_desc_num;
203
-	u64 rx_copybreak_pkt;
204234 	u64 bad_req_id;
205235 	u64 empty_rx_ring;
236
+	u64 csum_unchecked;
237
+	u64 xdp_aborted;
238
+	u64 xdp_drop;
239
+	u64 xdp_pass;
240
+	u64 xdp_tx;
241
+	u64 xdp_invalid;
206242 };
207243 
208244 struct ena_ring {
209
-	union {
210
-		/* Holds the empty requests for TX/RX
211
-		 * out of order completions
212
-		 */
213
-		u16 *free_tx_ids;
214
-		u16 *free_rx_ids;
215
-	};
245
+	/* Holds the empty requests for TX/RX
246
+	 * out of order completions
247
+	 */
248
+	u16 *free_ids;
216249 
217250 	union {
218251 		struct ena_tx_buffer *tx_buffer_info;
....@@ -228,10 +261,13 @@
228261 	struct ena_adapter *adapter;
229262 	struct ena_com_io_cq *ena_com_io_cq;
230263 	struct ena_com_io_sq *ena_com_io_sq;
264
+	struct bpf_prog *xdp_bpf_prog;
265
+	struct xdp_rxq_info xdp_rxq;
231266 
232267 	u16 next_to_use;
233268 	u16 next_to_clean;
234269 	u16 rx_copybreak;
270
+	u16 rx_headroom;
235271 	u16 qid;
236272 	u16 mtu;
237273 	u16 sgl_size;
....@@ -240,6 +276,7 @@
240276 	u8 tx_max_header_size;
241277 
242278 	bool first_interrupt;
279
+	bool disable_meta_caching;
243280 	u16 no_interrupt_event_cnt;
244281 
245282 	/* cpu for TPH */
....@@ -252,13 +289,14 @@
252289 	struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS];
253290 	u32  smoothed_interval;
254291 	u32  per_napi_packets;
255
-	u32  per_napi_bytes;
256
-	enum ena_intr_moder_level moder_tbl_idx;
292
+	u16 non_empty_napi_events;
257293 	struct u64_stats_sync syncp;
258294 	union {
259295 		struct ena_stats_tx tx_stats;
260296 		struct ena_stats_rx rx_stats;
261297 	};
298
+
299
+	u8 *push_buf_intermediate_buf;
262300 	int empty_rx_queue;
263301 } ____cacheline_aligned;
264302 
....@@ -271,6 +309,7 @@
271309 	u64 interface_down;
272310 	u64 admin_q_pause;
273311 	u64 rx_drops;
312
+	u64 tx_drops;
274313 };
275314 
276315 enum ena_flags_t {
....@@ -295,17 +334,18 @@
295334 	u32 rx_copybreak;
296335 	u32 max_mtu;
297336 
298
-	int num_queues;
337
+	u32 num_io_queues;
338
+	u32 max_num_io_queues;
299339 
300340 	int msix_vecs;
301341 
302342 	u32 missing_tx_completion_threshold;
303343 
304
-	u32 tx_usecs, rx_usecs; /* interrupt moderation */
305
-	u32 tx_frames, rx_frames; /* interrupt moderation */
344
+	u32 requested_tx_ring_size;
345
+	u32 requested_rx_ring_size;
306346 
307
-	u32 tx_ring_size;
308
-	u32 rx_ring_size;
347
+	u32 max_tx_ring_size;
348
+	u32 max_rx_ring_size;
309349 
310350 	u32 msg_enable;
311351 
....@@ -338,15 +378,22 @@
338378 
339379 	bool wd_state;
340380 	bool dev_up_before_reset;
381
+	bool disable_meta_caching;
341382 	unsigned long last_keep_alive_jiffies;
342383 
343384 	struct u64_stats_sync syncp;
344385 	struct ena_stats_dev dev_stats;
386
+	struct ena_admin_eni_stats eni_stats;
387
+	bool eni_stats_supported;
345388 
346389 	/* last queue index that was checked for uncompleted tx packets */
347390 	u32 last_monitored_tx_qid;
348391 
349392 	enum ena_regs_reset_reason_types reset_reason;
393
+
394
+	struct bpf_prog *xdp_bpf_prog;
395
+	u32 xdp_first_ring;
396
+	u32 xdp_num_queues;
350397 };
351398 
352399 void ena_set_ethtool_ops(struct net_device *netdev);
....@@ -355,17 +402,53 @@
355402 
356403 void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf);
357404 
405
+int ena_update_hw_stats(struct ena_adapter *adapter);
406
+
407
+int ena_update_queue_sizes(struct ena_adapter *adapter,
408
+			   u32 new_tx_size,
409
+			   u32 new_rx_size);
410
+
411
+int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count);
412
+
358413 int ena_get_sset_count(struct net_device *netdev, int sset);
359414 
360
-/* The ENA buffer length fields is 16 bit long. So when PAGE_SIZE == 64kB the
361
- * driver passas 0.
362
- * Since the max packet size the ENA handles is ~9kB limit the buffer length to
363
- * 16kB.
364
- */
365
-#if PAGE_SIZE > SZ_16K
366
-#define ENA_PAGE_SIZE SZ_16K
367
-#else
368
-#define ENA_PAGE_SIZE PAGE_SIZE
369
-#endif
415
+enum ena_xdp_errors_t {
416
+	ENA_XDP_ALLOWED = 0,
417
+	ENA_XDP_CURRENT_MTU_TOO_LARGE,
418
+	ENA_XDP_NO_ENOUGH_QUEUES,
419
+};
420
+
421
+static inline bool ena_xdp_queues_present(struct ena_adapter *adapter)
422
+{
423
+	return adapter->xdp_first_ring != 0;
424
+}
425
+
426
+static inline bool ena_xdp_present(struct ena_adapter *adapter)
427
+{
428
+	return !!adapter->xdp_bpf_prog;
429
+}
430
+
431
+static inline bool ena_xdp_present_ring(struct ena_ring *ring)
432
+{
433
+	return !!ring->xdp_bpf_prog;
434
+}
435
+
436
+static inline int ena_xdp_legal_queue_count(struct ena_adapter *adapter,
437
+					    u32 queues)
438
+{
439
+	return 2 * queues <= adapter->max_num_io_queues;
440
+}
441
+
442
+static inline enum ena_xdp_errors_t ena_xdp_allowed(struct ena_adapter *adapter)
443
+{
444
+	enum ena_xdp_errors_t rc = ENA_XDP_ALLOWED;
445
+
446
+	if (adapter->netdev->mtu > ENA_XDP_MAX_MTU)
447
+		rc = ENA_XDP_CURRENT_MTU_TOO_LARGE;
448
+	else if (!ena_xdp_legal_queue_count(adapter, adapter->num_io_queues))
449
+		rc = ENA_XDP_NO_ENOUGH_QUEUES;
450
+
451
+	return rc;
452
+}
370453 
371454 #endif /* !(ENA_H) */