forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/drivers/net/ethernet/sfc/net_driver.h
....@@ -1,11 +1,8 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 /****************************************************************************
23  * Driver for Solarflare network controllers and boards
34  * Copyright 2005-2006 Fen Systems Ltd.
45  * Copyright 2005-2013 Solarflare Communications Inc.
5
- *
6
- * This program is free software; you can redistribute it and/or modify it
7
- * under the terms of the GNU General Public License version 2 as published
8
- * by the Free Software Foundation, incorporated herein by reference.
96  */
107 
118 /* Common definitions for all Efx net driver code */
....@@ -27,9 +24,9 @@
2724 #include <linux/mutex.h>
2825 #include <linux/rwsem.h>
2926 #include <linux/vmalloc.h>
30
-#include <linux/i2c.h>
3127 #include <linux/mtd/mtd.h>
3228 #include <net/busy_poll.h>
29
+#include <net/xdp.h>
3330 
3431 #include "enum.h"
3532 #include "bitfield.h"
....@@ -40,8 +37,6 @@
4037  * Build definitions
4138  *
4239  **************************************************************************/
43
-
44
-#define EFX_DRIVER_VERSION	"4.1"
4540 
4641 #ifdef DEBUG
4742 #define EFX_WARN_ON_ONCE_PARANOID(x) WARN_ON_ONCE(x)
....@@ -68,16 +63,22 @@
6863  * queues. */
6964 #define EFX_MAX_TX_TC		2
7065 #define EFX_MAX_CORE_TX_QUEUES	(EFX_MAX_TX_TC * EFX_MAX_CHANNELS)
71
-#define EFX_TXQ_TYPE_OFFLOAD	1	/* flag */
72
-#define EFX_TXQ_TYPE_HIGHPRI	2	/* flag */
73
-#define EFX_TXQ_TYPES		4
74
-#define EFX_MAX_TX_QUEUES	(EFX_TXQ_TYPES * EFX_MAX_CHANNELS)
66
+#define EFX_TXQ_TYPE_OUTER_CSUM	1	/* Outer checksum offload */
67
+#define EFX_TXQ_TYPE_INNER_CSUM	2	/* Inner checksum offload */
68
+#define EFX_TXQ_TYPE_HIGHPRI	4	/* High-priority (for TC) */
69
+#define EFX_TXQ_TYPES		8
70
+/* HIGHPRI is Siena-only, and INNER_CSUM is EF10, so no need for both */
71
+#define EFX_MAX_TXQ_PER_CHANNEL	4
72
+#define EFX_MAX_TX_QUEUES	(EFX_MAX_TXQ_PER_CHANNEL * EFX_MAX_CHANNELS)
7573 
7674 /* Maximum possible MTU the driver supports */
7775 #define EFX_MAX_MTU (9 * 1024)
7876 
7977 /* Minimum MTU, from RFC791 (IP) */
8078 #define EFX_MIN_MTU 68
79
+
80
+/* Maximum total header length for TSOv2 */
81
+#define EFX_TSO2_MAX_HDRLEN	208
8182 
8283 /* Size of an RX scatter buffer.  Small enough to pack 2 into a 4K page,
8384  * and should be a multiple of the cache line size.
....@@ -93,6 +94,12 @@
9394 #else
9495 #define EFX_RX_BUF_ALIGNMENT	4
9596 #endif
97
+
98
+/* Non-standard XDP_PACKET_HEADROOM and tailroom to satisfy XDP_REDIRECT and
99
+ * still fit two standard MTU size packets into a single 4K page.
100
+ */
101
+#define EFX_XDP_HEADROOM	128
102
+#define EFX_XDP_TAILROOM	SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
96103 
97104 /* Forward declare Precision Time Protocol (PTP) support structure. */
98105 struct efx_ptp_data;
....@@ -139,7 +146,10 @@
139146  * struct efx_tx_buffer - buffer state for a TX descriptor
140147  * @skb: When @flags & %EFX_TX_BUF_SKB, the associated socket buffer to be
141148  *	freed when descriptor completes
142
- * @option: When @flags & %EFX_TX_BUF_OPTION, a NIC-specific option descriptor.
149
+ * @xdpf: When @flags & %EFX_TX_BUF_XDP, the XDP frame information; its @data
150
+ *	member is the associated buffer to drop a page reference on.
151
+ * @option: When @flags & %EFX_TX_BUF_OPTION, an EF10-specific option
152
+ *	descriptor.
143153  * @dma_addr: DMA address of the fragment.
144154  * @flags: Flags for allocation and DMA mapping type
145155  * @len: Length of this fragment.
....@@ -149,9 +159,12 @@
149159  * Only valid if @unmap_len != 0.
150160  */
151161 struct efx_tx_buffer {
152
-	const struct sk_buff *skb;
153162 	union {
154
-		efx_qword_t option;
163
+		const struct sk_buff *skb;
164
+		struct xdp_frame *xdpf;
165
+	};
166
+	union {
167
+		efx_qword_t option;    /* EF10 */
155168 		dma_addr_t dma_addr;
156169 	};
157170 	unsigned short flags;
....@@ -163,6 +176,8 @@
163176 #define EFX_TX_BUF_SKB		2	/* buffer is last part of skb */
164177 #define EFX_TX_BUF_MAP_SINGLE	8	/* buffer was mapped with dma_map_single() */
165178 #define EFX_TX_BUF_OPTION	0x10	/* empty buffer for option descriptor */
179
+#define EFX_TX_BUF_XDP		0x20	/* buffer was sent with XDP */
180
+#define EFX_TX_BUF_TSO_V3	0x40	/* empty buffer for a TSO_V3 descriptor */
166181 
167182 /**
168183  * struct efx_tx_queue - An Efx TX queue
....@@ -179,7 +194,11 @@
179194  *
180195  * @efx: The associated Efx NIC
181196  * @queue: DMA queue number
197
+ * @label: Label for TX completion events.
198
+ *	Is our index within @channel->tx_queue array.
199
+ * @type: configuration type of this TX queue.  A bitmask of %EFX_TXQ_TYPE_* flags.
182200  * @tso_version: Version of TSO in use for this queue.
201
+ * @tso_encap: Is encapsulated TSO supported? Supported in TSOv2 on 8000 series.
183202  * @channel: The associated channel
184203  * @core_txq: The networking core TX queue structure
185204  * @buffer: The software buffer ring
....@@ -192,8 +211,7 @@
192211  * @piobuf_offset: Buffer offset to be specified in PIO descriptors
193212  * @initialised: Has hardware queue been initialised?
194213  * @timestamping: Is timestamping enabled for this channel?
195
- * @handle_tso: TSO xmit preparation handler.  Sets up the TSO metadata and
196
- *	may also map tx data, depending on the nature of the TSO implementation.
214
+ * @xdp_tx: Is this an XDP tx queue?
197215  * @read_count: Current read pointer.
198216  *	This is the number of buffers that have been removed from both rings.
199217  * @old_write_count: The value of @write_count when last checked.
....@@ -203,8 +221,6 @@
203221  *	avoid cache-line ping-pong between the xmit path and the
204222  *	completion path.
205223  * @merge_events: Number of TX merged completion events
206
- * @completed_desc_ptr: Most recent completed pointer - only used with
207
- *      timestamping.
208224  * @completed_timestamp_major: Top part of the most recent tx timestamp.
209225  * @completed_timestamp_minor: Low part of the most recent tx timestamp.
210226  * @insert_count: Current insert pointer
....@@ -232,8 +248,9 @@
232248  * @tso_fallbacks: Number of times TSO fallback used
233249  * @pushes: Number of times the TX push feature has been used
234250  * @pio_packets: Number of times the TX PIO feature has been used
235
- * @xmit_more_available: Are any packets waiting to be pushed to the NIC
251
+ * @xmit_pending: Are any packets waiting to be pushed to the NIC
236252  * @cb_packets: Number of times the TX copybreak feature has been used
253
+ * @notify_count: Count of notified descriptors to the NIC
237254  * @empty_read_count: If the completion path has seen the queue as empty
238255  *	and the transmission path has not yet checked this, the value of
239256  *	@read_count bitwise-added to %EFX_EMPTY_COUNT_VALID; otherwise 0.
....@@ -241,8 +258,11 @@
241258 struct efx_tx_queue {
242259 	/* Members which don't change on the fast path */
243260 	struct efx_nic *efx ____cacheline_aligned_in_smp;
244
-	unsigned queue;
261
+	unsigned int queue;
262
+	unsigned int label;
263
+	unsigned int type;
245264 	unsigned int tso_version;
265
+	bool tso_encap;
246266 	struct efx_channel *channel;
247267 	struct netdev_queue *core_txq;
248268 	struct efx_tx_buffer *buffer;
....@@ -253,9 +273,7 @@
253273 	unsigned int piobuf_offset;
254274 	bool initialised;
255275 	bool timestamping;
256
-
257
-	/* Function pointers used in the fast path. */
258
-	int (*handle_tso)(struct efx_tx_queue*, struct sk_buff*, bool *);
276
+	bool xdp_tx;
259277 
260278 	/* Members used mainly on the completion path */
261279 	unsigned int read_count ____cacheline_aligned_in_smp;
....@@ -263,7 +281,6 @@
263281 	unsigned int merge_events;
264282 	unsigned int bytes_compl;
265283 	unsigned int pkts_compl;
266
-	unsigned int completed_desc_ptr;
267284 	u32 completed_timestamp_major;
268285 	u32 completed_timestamp_minor;
269286 
....@@ -278,8 +295,9 @@
278295 	unsigned int tso_fallbacks;
279296 	unsigned int pushes;
280297 	unsigned int pio_packets;
281
-	bool xmit_more_available;
298
+	bool xmit_pending;
282299 	unsigned int cb_packets;
300
+	unsigned int notify_count;
283301 	/* Statistics to supplement MAC stats */
284302 	unsigned long tx_packets;
285303 
....@@ -330,7 +348,7 @@
330348 struct efx_rx_page_state {
331349 	dma_addr_t dma_addr;
332350 
333
-	unsigned int __pad[0] ____cacheline_aligned;
351
+	unsigned int __pad[] ____cacheline_aligned;
334352 };
335353 
336354 /**
....@@ -366,6 +384,8 @@
366384  *	refill was triggered.
367385  * @recycle_count: RX buffer recycle counter.
368386  * @slow_fill: Timer used to defer efx_nic_generate_fill_event().
387
+ * @xdp_rxq_info: XDP specific RX queue information.
388
+ * @xdp_rxq_info_valid: Is xdp_rxq_info valid data?.
369389  */
370390 struct efx_rx_queue {
371391 	struct efx_nic *efx;
....@@ -397,6 +417,8 @@
397417 	unsigned int slow_fill_count;
398418 	/* Statistics to supplement MAC stats */
399419 	unsigned long rx_packets;
420
+	struct xdp_rxq_info xdp_rxq_info;
421
+	bool xdp_rxq_info_valid;
400422 };
401423 
402424 enum efx_sync_events_state {
....@@ -430,6 +452,13 @@
430452  * @event_test_cpu: Last CPU to handle interrupt or test event for this channel
431453  * @irq_count: Number of IRQs since last adaptive moderation decision
432454  * @irq_mod_score: IRQ moderation score
455
+ * @rfs_filter_count: number of accelerated RFS filters currently in place;
456
+ *	equals the count of @rps_flow_id slots filled
457
+ * @rfs_last_expiry: value of jiffies last time some accelerated RFS filters
458
+ *	were checked for expiry
459
+ * @rfs_expire_index: next accelerated RFS filter ID to check for expiry
460
+ * @n_rfs_succeeded: number of successful accelerated RFS filter insertions
461
+ * @n_rfs_failed: number of failed accelerated RFS filter insertions
433462  * @filter_work: Work item for efx_filter_rfs_expire()
434463  * @rps_flow_id: Flow IDs of filters allocated for accelerated RFS,
435464  *      indexed by filter ID
....@@ -444,6 +473,10 @@
444473  *	lack of descriptors
445474  * @n_rx_merge_events: Number of RX merged completion events
446475  * @n_rx_merge_packets: Number of RX packets completed by merged events
476
+ * @n_rx_xdp_drops: Count of RX packets intentionally dropped due to XDP
477
+ * @n_rx_xdp_bad_drops: Count of RX packets dropped due to XDP errors
478
+ * @n_rx_xdp_tx: Count of RX packets retransmitted due to XDP
479
+ * @n_rx_xdp_redirect: Count of RX packets redirected to a different NIC by XDP
447480  * @rx_pkt_n_frags: Number of fragments in next packet to be delivered by
448481  *	__efx_rx_packet(), or zero if there is none
449482  * @rx_pkt_index: Ring index of first buffer for next packet to be delivered
....@@ -451,6 +484,7 @@
451484  * @rx_list: list of SKBs from current RX, awaiting processing
452485  * @rx_queue: RX queue for this channel
453486  * @tx_queue: TX queues for this channel
487
+ * @tx_queue_by_type: pointers into @tx_queue, or %NULL, indexed by txq type
454488  * @sync_events_state: Current state of sync events on this channel
455489  * @sync_timestamp_major: Major part of the last ptp sync event
456490  * @sync_timestamp_minor: Minor part of the last ptp sync event
....@@ -476,8 +510,12 @@
476510 	unsigned int irq_count;
477511 	unsigned int irq_mod_score;
478512 #ifdef CONFIG_RFS_ACCEL
479
-	unsigned int rfs_filters_added;
480
-	struct work_struct filter_work;
513
+	unsigned int rfs_filter_count;
514
+	unsigned int rfs_last_expiry;
515
+	unsigned int rfs_expire_index;
516
+	unsigned int n_rfs_succeeded;
517
+	unsigned int n_rfs_failed;
518
+	struct delayed_work filter_work;
481519 #define RPS_FLOW_ID_INVALID 0xFFFFFFFF
482520 	u32 *rps_flow_id;
483521 #endif
....@@ -497,6 +535,10 @@
497535 	unsigned int n_rx_nodesc_trunc;
498536 	unsigned int n_rx_merge_events;
499537 	unsigned int n_rx_merge_packets;
538
+	unsigned int n_rx_xdp_drops;
539
+	unsigned int n_rx_xdp_bad_drops;
540
+	unsigned int n_rx_xdp_tx;
541
+	unsigned int n_rx_xdp_redirect;
500542 
501543 	unsigned int rx_pkt_n_frags;
502544 	unsigned int rx_pkt_index;
....@@ -504,7 +546,8 @@
504546 	struct list_head *rx_list;
505547 
506548 	struct efx_rx_queue rx_queue;
507
-	struct efx_tx_queue tx_queue[EFX_TXQ_TYPES];
549
+	struct efx_tx_queue tx_queue[EFX_MAX_TXQ_PER_CHANNEL];
550
+	struct efx_tx_queue *tx_queue_by_type[EFX_TXQ_TYPES];
508551 
509552 	enum efx_sync_events_state sync_events_state;
510553 	u32 sync_timestamp_major;
....@@ -574,8 +617,6 @@
574617 #define RESET_TYPE(type) \
575618 	STRING_TABLE_LOOKUP(type, efx_reset_type)
576619 
577
-void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen);
578
-
579620 enum efx_int_mode {
580621 	/* Be careful if altering to correct macro below */
581622 	EFX_INT_MODE_MSIX = 0,
....@@ -620,51 +661,6 @@
620661 	return left->up == right->up && left->fd == right->fd &&
621662 		left->fc == right->fc && left->speed == right->speed;
622663 }
623
-
624
-/**
625
- * struct efx_phy_operations - Efx PHY operations table
626
- * @probe: Probe PHY and initialise efx->mdio.mode_support, efx->mdio.mmds,
627
- *	efx->loopback_modes.
628
- * @init: Initialise PHY
629
- * @fini: Shut down PHY
630
- * @reconfigure: Reconfigure PHY (e.g. for new link parameters)
631
- * @poll: Update @link_state and report whether it changed.
632
- *	Serialised by the mac_lock.
633
- * @get_link_ksettings: Get ethtool settings. Serialised by the mac_lock.
634
- * @set_link_ksettings: Set ethtool settings. Serialised by the mac_lock.
635
- * @get_fecparam: Get Forward Error Correction settings. Serialised by mac_lock.
636
- * @set_fecparam: Set Forward Error Correction settings. Serialised by mac_lock.
637
- * @set_npage_adv: Set abilities advertised in (Extended) Next Page
638
- *	(only needed where AN bit is set in mmds)
639
- * @test_alive: Test that PHY is 'alive' (online)
640
- * @test_name: Get the name of a PHY-specific test/result
641
- * @run_tests: Run tests and record results as appropriate (offline).
642
- *	Flags are the ethtool tests flags.
643
- */
644
-struct efx_phy_operations {
645
-	int (*probe) (struct efx_nic *efx);
646
-	int (*init) (struct efx_nic *efx);
647
-	void (*fini) (struct efx_nic *efx);
648
-	void (*remove) (struct efx_nic *efx);
649
-	int (*reconfigure) (struct efx_nic *efx);
650
-	bool (*poll) (struct efx_nic *efx);
651
-	void (*get_link_ksettings)(struct efx_nic *efx,
652
-				   struct ethtool_link_ksettings *cmd);
653
-	int (*set_link_ksettings)(struct efx_nic *efx,
654
-				  const struct ethtool_link_ksettings *cmd);
655
-	int (*get_fecparam)(struct efx_nic *efx, struct ethtool_fecparam *fec);
656
-	int (*set_fecparam)(struct efx_nic *efx,
657
-			    const struct ethtool_fecparam *fec);
658
-	void (*set_npage_adv) (struct efx_nic *efx, u32);
659
-	int (*test_alive) (struct efx_nic *efx);
660
-	const char *(*test_name) (struct efx_nic *efx, unsigned int index);
661
-	int (*run_tests) (struct efx_nic *efx, int *results, unsigned flags);
662
-	int (*get_module_eeprom) (struct efx_nic *efx,
663
-			       struct ethtool_eeprom *ee,
664
-			       u8 *data);
665
-	int (*get_module_info) (struct efx_nic *efx,
666
-				struct ethtool_modinfo *modinfo);
667
-};
668664 
669665 /**
670666  * enum efx_phy_mode - PHY operating mode flags
....@@ -715,13 +711,13 @@
715711 struct vfdi_status;
716712 
717713 /* The reserved RSS context value */
718
-#define EFX_EF10_RSS_CONTEXT_INVALID	0xffffffff
714
+#define EFX_MCDI_RSS_CONTEXT_INVALID	0xffffffff
719715 /**
720716  * struct efx_rss_context - A user-defined RSS context for filtering
721717  * @list: node of linked list on which this struct is stored
722718  * @context_id: the RSS_CONTEXT_ID returned by MC firmware, or
723
- *	%EFX_EF10_RSS_CONTEXT_INVALID if this context is not present on the NIC.
724
- *	For Siena, 0 if RSS is active, else %EFX_EF10_RSS_CONTEXT_INVALID.
719
+ *	%EFX_MCDI_RSS_CONTEXT_INVALID if this context is not present on the NIC.
720
+ *	For Siena, 0 if RSS is active, else %EFX_MCDI_RSS_CONTEXT_INVALID.
725721  * @user_id: the rss_context ID exposed to userspace over ethtool.
726722  * @rx_hash_udp_4tuple: UDP 4-tuple hashing enabled
727723  * @rx_hash_key: Toeplitz hash key for this RSS context
....@@ -810,6 +806,7 @@
810806  * @timer_quantum_ns: Interrupt timer quantum, in nanoseconds
811807  * @timer_max_ns: Interrupt timer maximum value, in nanoseconds
812808  * @irq_rx_adaptive: Adaptive IRQ moderation enabled for RX event queues
809
+ * @irqs_hooked: Channel interrupts are hooked
813810  * @irq_rx_mod_step_us: Step size for IRQ moderation for RX event queues
814811  * @irq_rx_moderation_us: IRQ moderation time for RX event queues
815812  * @msg_enable: Log message enable flags
....@@ -821,6 +818,8 @@
821818  * @msi_context: Context for each MSI
822819  * @extra_channel_types: Types of extra (non-traffic) channels that
823820  *	should be allocated for this NIC
821
+ * @xdp_tx_queue_count: Number of entries in %xdp_tx_queues.
822
+ * @xdp_tx_queues: Array of pointers to tx queues used for XDP transmit.
824823  * @rxq_entries: Size of receive queues requested by user.
825824  * @txq_entries: Size of transmit queues requested by user.
826825  * @txq_stop_thresh: TX queue fill level at or above which we stop it.
....@@ -833,6 +832,10 @@
833832  * @n_rx_channels: Number of channels used for RX (= number of RX queues)
834833  * @n_tx_channels: Number of channels used for TX
835834  * @n_extra_tx_channels: Number of extra channels with TX queues
835
+ * @tx_queues_per_channel: number of TX queues probed on each channel
836
+ * @n_xdp_channels: Number of channels used for XDP TX
837
+ * @xdp_channel_offset: Offset of zeroth channel used for XPD TX.
838
+ * @xdp_tx_per_channel: Max number of TX queues on an XDP TX channel.
836839  * @rx_ip_align: RX DMA address offset to have IP header aligned in
837840  *	in accordance with NET_IP_ALIGN
838841  * @rx_dma_len: Current maximum RX DMA length
....@@ -850,8 +853,10 @@
850853  * @rss_context: Main RSS context.  Its @list member is the head of the list of
851854  *	RSS contexts created by user requests
852855  * @rss_lock: Protects custom RSS context software state in @rss_context.list
856
+ * @vport_id: The function's vport ID, only relevant for PFs
853857  * @int_error_count: Number of internal errors seen recently
854858  * @int_error_expire: Time at which error count will be expired
859
+ * @must_realloc_vis: Flag: VIs have yet to be reallocated after MC reboot
855860  * @irq_soft_enabled: Are IRQs soft-enabled? If not, IRQ handler will
856861  *	acknowledge but do nothing else.
857862  * @irq_status: Interrupt status buffer
....@@ -875,7 +880,6 @@
875880  *	field of %MC_CMD_GET_CAPABILITIES_V4 response, or %MC_CMD_MAC_NSTATS)
876881  * @stats_buffer: DMA buffer for statistics
877882  * @phy_type: PHY type
878
- * @phy_op: PHY interface
879883  * @phy_data: PHY private data (including PHY-specific stats)
880884  * @mdio: PHY MDIO interface
881885  * @mdio_bus: PHY MDIO bus ID (only used by Siena)
....@@ -897,12 +901,10 @@
897901  * @loopback_mode: Loopback status
898902  * @loopback_modes: Supported loopback mode bitmask
899903  * @loopback_selftest: Offline self-test private state
904
+ * @xdp_prog: Current XDP programme for this interface
900905  * @filter_sem: Filter table rw_semaphore, protects existence of @filter_state
901906  * @filter_state: Architecture-dependent filter table state
902907  * @rps_mutex: Protects RPS state of all channels
903
- * @rps_expire_channel: Next channel to check for expiry
904
- * @rps_expire_index: Next index to check for expiry in
905
- *	@rps_expire_channel's @rps_flow_id
906908  * @rps_slot_map: bitmap of in-flight entries in @rps_slot
907909  * @rps_slot: array of ARFS insertion requests for efx_filter_rfs_work()
908910  * @rps_hash_lock: Protects ARFS filter mapping state (@rps_hash_table and
....@@ -922,6 +924,11 @@
922924  * @ptp_data: PTP state data
923925  * @ptp_warned: has this NIC seen and warned about unexpected PTP events?
924926  * @vpd_sn: Serial number read from VPD
927
+ * @xdp_rxq_info_failed: Have any of the rx queues failed to initialise their
928
+ *      xdp_rxq_info structures?
929
+ * @netdev_notifier: Netdevice notifier.
930
+ * @mem_bar: The BAR that is mapped into membase.
931
+ * @reg_base: Offset from the start of the bar to the function control window.
925932  * @monitor_work: Hardware monitor workitem
926933  * @biu_lock: BIU (bus interface unit) lock
927934  * @last_irq_cpu: Last CPU to handle a possible test interrupt.  This
....@@ -957,6 +964,7 @@
957964 	unsigned int timer_quantum_ns;
958965 	unsigned int timer_max_ns;
959966 	bool irq_rx_adaptive;
967
+	bool irqs_hooked;
960968 	unsigned int irq_mod_step_us;
961969 	unsigned int irq_rx_moderation_us;
962970 	u32 msg_enable;
....@@ -969,6 +977,9 @@
969977 	const struct efx_channel_type *
970978 	extra_channel_type[EFX_MAX_EXTRA_CHANNELS];
971979 
980
+	unsigned int xdp_tx_queue_count;
981
+	struct efx_tx_queue **xdp_tx_queues;
982
+
972983 	unsigned rxq_entries;
973984 	unsigned txq_entries;
974985 	unsigned int txq_stop_thresh;
....@@ -980,6 +991,7 @@
980991 	unsigned next_buffer_table;
981992 
982993 	unsigned int max_channels;
994
+	unsigned int max_vis;
983995 	unsigned int max_tx_channels;
984996 	unsigned n_channels;
985997 	unsigned n_rx_channels;
....@@ -987,6 +999,10 @@
987999 	unsigned tx_channel_offset;
9881000 	unsigned n_tx_channels;
9891001 	unsigned n_extra_tx_channels;
1002
+	unsigned int tx_queues_per_channel;
1003
+	unsigned int n_xdp_channels;
1004
+	unsigned int xdp_channel_offset;
1005
+	unsigned int xdp_tx_per_channel;
9901006 	unsigned int rx_ip_align;
9911007 	unsigned int rx_dma_len;
9921008 	unsigned int rx_buffer_order;
....@@ -1001,10 +1017,12 @@
10011017 	bool rx_scatter;
10021018 	struct efx_rss_context rss_context;
10031019 	struct mutex rss_lock;
1020
+	u32 vport_id;
10041021 
10051022 	unsigned int_error_count;
10061023 	unsigned long int_error_expire;
10071024 
1025
+	bool must_realloc_vis;
10081026 	bool irq_soft_enabled;
10091027 	struct efx_buffer irq_status;
10101028 	unsigned irq_zero_count;
....@@ -1035,7 +1053,6 @@
10351053 	bool rx_nodesc_drops_prev_state;
10361054 
10371055 	unsigned int phy_type;
1038
-	const struct efx_phy_operations *phy_op;
10391056 	void *phy_data;
10401057 	struct mdio_if_info mdio;
10411058 	unsigned int mdio_bus;
....@@ -1056,13 +1073,15 @@
10561073 	u64 loopback_modes;
10571074 
10581075 	void *loopback_selftest;
1076
+	/* We access loopback_selftest immediately before running XDP,
1077
+	 * so we want them next to each other.
1078
+	 */
1079
+	struct bpf_prog __rcu *xdp_prog;
10591080 
10601081 	struct rw_semaphore filter_sem;
10611082 	void *filter_state;
10621083 #ifdef CONFIG_RFS_ACCEL
10631084 	struct mutex rps_mutex;
1064
-	unsigned int rps_expire_channel;
1065
-	unsigned int rps_expire_index;
10661085 	unsigned long rps_slot_map;
10671086 	struct efx_async_filter_insertion rps_slot[EFX_RPS_MAX_IN_FLIGHT];
10681087 	spinlock_t rps_hash_lock;
....@@ -1085,6 +1104,12 @@
10851104 	bool ptp_warned;
10861105 
10871106 	char *vpd_sn;
1107
+	bool xdp_rxq_info_failed;
1108
+
1109
+	struct notifier_block netdev_notifier;
1110
+
1111
+	unsigned int mem_bar;
1112
+	u32 reg_base;
10881113 
10891114 	/* The following fields may be written more often */
10901115 
....@@ -1114,12 +1139,9 @@
11141139 };
11151140 
11161141 struct efx_udp_tunnel {
1142
+#define TUNNEL_ENCAP_UDP_PORT_ENTRY_INVALID	0xffff
11171143 	u16 type; /* TUNNEL_ENCAP_UDP_PORT_ENTRY_foo, see mcdi_pcol.h */
11181144 	__be16 port;
1119
-	/* Count of repeated adds of the same port.  Used only inside the list,
1120
-	 * not in request arguments.
1121
-	 */
1122
-	u16 count;
11231145 };
11241146 
11251147 /**
....@@ -1150,10 +1172,12 @@
11501172  * @describe_stats: Describe statistics for ethtool
11511173  * @update_stats: Update statistics not provided by event handling.
11521174  *	Either argument may be %NULL.
1175
+ * @update_stats_atomic: Update statistics while in atomic context, if that
1176
+ *	is more limiting than @update_stats.  Otherwise, leave %NULL and
1177
+ *	driver core will call @update_stats.
11531178  * @start_stats: Start the regular fetching of statistics
11541179  * @pull_stats: Pull stats from the NIC and wait until they arrive.
11551180  * @stop_stats: Stop the regular fetching of statistics
1156
- * @set_id_led: Set state of identifying LED or revert to automatic function
11571181  * @push_irq_moderation: Apply interrupt moderation value
11581182  * @reconfigure_port: Push loopback/power/txdis changes to the MAC and PHY
11591183  * @prepare_enable_fc_tx: Prepare MAC to enable pause frame TX (may be %NULL)
....@@ -1186,10 +1210,11 @@
11861210  *	a pointer to the &struct efx_msi_context for the channel.
11871211  * @irq_handle_legacy: Handle legacy interrupt.  The @dev_id argument
11881212  *	is a pointer to the &struct efx_nic.
1189
- * @tx_probe: Allocate resources for TX queue
1213
+ * @tx_probe: Allocate resources for TX queue (and select TXQ type)
11901214  * @tx_init: Initialise TX queue on the NIC
11911215  * @tx_remove: Free resources for TX queue
11921216  * @tx_write: Write TX descriptors and doorbell
1217
+ * @tx_enqueue: Add an SKB to TX queue
11931218  * @rx_push_rss_config: Write RSS hash key and indirection table to the NIC
11941219  * @rx_pull_rss_config: Read RSS hash key and indirection table back from the NIC
11951220  * @rx_push_rss_context_config: Write RSS hash key and indirection table for
....@@ -1201,6 +1226,8 @@
12011226  * @rx_remove: Free resources for RX queue
12021227  * @rx_write: Write RX descriptors and doorbell
12031228  * @rx_defer_refill: Generate a refill reminder event
1229
+ * @rx_packet: Receive the queued RX buffer on a channel
1230
+ * @rx_buf_hash_valid: Determine whether the RX prefix contains a valid hash
12041231  * @ev_probe: Allocate resources for event queue
12051232  * @ev_init: Initialise event queue on the NIC
12061233  * @ev_fini: Deinitialise event queue on the NIC
....@@ -1243,9 +1270,9 @@
12431270  * @tso_versions: Returns mask of firmware-assisted TSO versions supported.
12441271  *	If %NULL, then device does not support any TSO version.
12451272  * @udp_tnl_push_ports: Push the list of UDP tunnel ports to the NIC if required.
1246
- * @udp_tnl_add_port: Add a UDP tunnel port
12471273  * @udp_tnl_has_port: Check if a port has been added as UDP tunnel
1248
- * @udp_tnl_del_port: Remove a UDP tunnel port
1274
+ * @print_additional_fwver: Dump NIC-specific additional FW version info
1275
+ * @sensor_event: Handle a sensor event from MCDI
12491276  * @revision: Hardware architecture revision
12501277  * @txd_ptr_tbl_base: TX descriptor ring base address
12511278  * @rxd_ptr_tbl_base: RX descriptor ring base address
....@@ -1261,8 +1288,6 @@
12611288  * @always_rx_scatter: NIC will always scatter packets to multiple buffers
12621289  * @option_descriptors: NIC supports TX option descriptors
12631290  * @min_interrupt_mode: Lowest capability interrupt mode supported
1264
- *	from &enum efx_int_mode.
1265
- * @max_interrupt_mode: Highest capability interrupt mode supported
12661291  *	from &enum efx_int_mode.
12671292  * @timer_period_max: Maximum period of interrupt timer (in ticks)
12681293  * @offload_features: net_device feature flags for protocol offload
....@@ -1294,18 +1319,22 @@
12941319 	size_t (*describe_stats)(struct efx_nic *efx, u8 *names);
12951320 	size_t (*update_stats)(struct efx_nic *efx, u64 *full_stats,
12961321 			       struct rtnl_link_stats64 *core_stats);
1322
+	size_t (*update_stats_atomic)(struct efx_nic *efx, u64 *full_stats,
1323
+				      struct rtnl_link_stats64 *core_stats);
12971324 	void (*start_stats)(struct efx_nic *efx);
12981325 	void (*pull_stats)(struct efx_nic *efx);
12991326 	void (*stop_stats)(struct efx_nic *efx);
1300
-	void (*set_id_led)(struct efx_nic *efx, enum efx_led_mode mode);
13011327 	void (*push_irq_moderation)(struct efx_channel *channel);
13021328 	int (*reconfigure_port)(struct efx_nic *efx);
13031329 	void (*prepare_enable_fc_tx)(struct efx_nic *efx);
1304
-	int (*reconfigure_mac)(struct efx_nic *efx);
1330
+	int (*reconfigure_mac)(struct efx_nic *efx, bool mtu_only);
13051331 	bool (*check_mac_fault)(struct efx_nic *efx);
13061332 	void (*get_wol)(struct efx_nic *efx, struct ethtool_wolinfo *wol);
13071333 	int (*set_wol)(struct efx_nic *efx, u32 type);
13081334 	void (*resume_wol)(struct efx_nic *efx);
1335
+	unsigned int (*check_caps)(const struct efx_nic *efx,
1336
+				   u8 flag,
1337
+				   u32 offset);
13091338 	int (*test_chip)(struct efx_nic *efx, struct efx_self_tests *tests);
13101339 	int (*test_nvram)(struct efx_nic *efx);
13111340 	void (*mcdi_request)(struct efx_nic *efx,
....@@ -1325,6 +1354,7 @@
13251354 	void (*tx_init)(struct efx_tx_queue *tx_queue);
13261355 	void (*tx_remove)(struct efx_tx_queue *tx_queue);
13271356 	void (*tx_write)(struct efx_tx_queue *tx_queue);
1357
+	netdev_tx_t (*tx_enqueue)(struct efx_tx_queue *tx_queue, struct sk_buff *skb);
13281358 	unsigned int (*tx_limit_len)(struct efx_tx_queue *tx_queue,
13291359 				     dma_addr_t dma_addr, unsigned int len);
13301360 	int (*rx_push_rss_config)(struct efx_nic *efx, bool user,
....@@ -1342,6 +1372,8 @@
13421372 	void (*rx_remove)(struct efx_rx_queue *rx_queue);
13431373 	void (*rx_write)(struct efx_rx_queue *rx_queue);
13441374 	void (*rx_defer_refill)(struct efx_rx_queue *rx_queue);
1375
+	void (*rx_packet)(struct efx_channel *channel);
1376
+	bool (*rx_buf_hash_valid)(const u8 *prefix);
13451377 	int (*ev_probe)(struct efx_channel *channel);
13461378 	int (*ev_init)(struct efx_channel *channel);
13471379 	void (*ev_fini)(struct efx_channel *channel);
....@@ -1413,9 +1445,10 @@
14131445 	int (*set_mac_address)(struct efx_nic *efx);
14141446 	u32 (*tso_versions)(struct efx_nic *efx);
14151447 	int (*udp_tnl_push_ports)(struct efx_nic *efx);
1416
-	int (*udp_tnl_add_port)(struct efx_nic *efx, struct efx_udp_tunnel tnl);
14171448 	bool (*udp_tnl_has_port)(struct efx_nic *efx, __be16 port);
1418
-	int (*udp_tnl_del_port)(struct efx_nic *efx, struct efx_udp_tunnel tnl);
1449
+	size_t (*print_additional_fwver)(struct efx_nic *efx, char *buf,
1450
+					 size_t len);
1451
+	void (*sensor_event)(struct efx_nic *efx, efx_qword_t *ev);
14191452 
14201453 	int revision;
14211454 	unsigned int txd_ptr_tbl_base;
....@@ -1432,7 +1465,6 @@
14321465 	bool always_rx_scatter;
14331466 	bool option_descriptors;
14341467 	unsigned int min_interrupt_mode;
1435
-	unsigned int max_interrupt_mode;
14361468 	unsigned int timer_period_max;
14371469 	netdev_features_t offload_features;
14381470 	int mcdi_max_ver;
....@@ -1468,32 +1500,51 @@
14681500 	     _channel = _channel->channel ?				\
14691501 		     (_efx)->channel[_channel->channel - 1] : NULL)
14701502 
1471
-static inline struct efx_tx_queue *
1472
-efx_get_tx_queue(struct efx_nic *efx, unsigned index, unsigned type)
1503
+static inline struct efx_channel *
1504
+efx_get_tx_channel(struct efx_nic *efx, unsigned int index)
14731505 {
1474
-	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_tx_channels ||
1475
-				  type >= EFX_TXQ_TYPES);
1476
-	return &efx->channel[efx->tx_channel_offset + index]->tx_queue[type];
1506
+	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_tx_channels);
1507
+	return efx->channel[efx->tx_channel_offset + index];
1508
+}
1509
+
1510
+static inline struct efx_channel *
1511
+efx_get_xdp_channel(struct efx_nic *efx, unsigned int index)
1512
+{
1513
+	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_xdp_channels);
1514
+	return efx->channel[efx->xdp_channel_offset + index];
1515
+}
1516
+
1517
+static inline bool efx_channel_is_xdp_tx(struct efx_channel *channel)
1518
+{
1519
+	return channel->channel - channel->efx->xdp_channel_offset <
1520
+	       channel->efx->n_xdp_channels;
14771521 }
14781522 
14791523 static inline bool efx_channel_has_tx_queues(struct efx_channel *channel)
14801524 {
1481
-	return channel->type && channel->type->want_txqs &&
1482
-				channel->type->want_txqs(channel);
1525
+	return channel && channel->channel >= channel->efx->tx_channel_offset;
1526
+}
1527
+
1528
+static inline unsigned int efx_channel_num_tx_queues(struct efx_channel *channel)
1529
+{
1530
+	if (efx_channel_is_xdp_tx(channel))
1531
+		return channel->efx->xdp_tx_per_channel;
1532
+	return channel->efx->tx_queues_per_channel;
14831533 }
14841534 
14851535 static inline struct efx_tx_queue *
1486
-efx_channel_get_tx_queue(struct efx_channel *channel, unsigned type)
1536
+efx_channel_get_tx_queue(struct efx_channel *channel, unsigned int type)
14871537 {
1488
-	EFX_WARN_ON_ONCE_PARANOID(!efx_channel_has_tx_queues(channel) ||
1489
-				  type >= EFX_TXQ_TYPES);
1490
-	return &channel->tx_queue[type];
1538
+	EFX_WARN_ON_ONCE_PARANOID(type >= EFX_TXQ_TYPES);
1539
+	return channel->tx_queue_by_type[type];
14911540 }
14921541 
1493
-static inline bool efx_tx_queue_used(struct efx_tx_queue *tx_queue)
1542
+static inline struct efx_tx_queue *
1543
+efx_get_tx_queue(struct efx_nic *efx, unsigned int index, unsigned int type)
14941544 {
1495
-	return !(tx_queue->efx->net_dev->num_tc < 2 &&
1496
-		 tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI);
1545
+	struct efx_channel *channel = efx_get_tx_channel(efx, index);
1546
+
1547
+	return efx_channel_get_tx_queue(channel, type);
14971548 }
14981549 
14991550 /* Iterate over all TX queues belonging to a channel */
....@@ -1502,17 +1553,8 @@
15021553 		;							\
15031554 	else								\
15041555 		for (_tx_queue = (_channel)->tx_queue;			\
1505
-		     _tx_queue < (_channel)->tx_queue + EFX_TXQ_TYPES && \
1506
-			     efx_tx_queue_used(_tx_queue);		\
1507
-		     _tx_queue++)
1508
-
1509
-/* Iterate over all possible TX queues belonging to a channel */
1510
-#define efx_for_each_possible_channel_tx_queue(_tx_queue, _channel)	\
1511
-	if (!efx_channel_has_tx_queues(_channel))			\
1512
-		;							\
1513
-	else								\
1514
-		for (_tx_queue = (_channel)->tx_queue;			\
1515
-		     _tx_queue < (_channel)->tx_queue + EFX_TXQ_TYPES;	\
1556
+		     _tx_queue < (_channel)->tx_queue +			\
1557
+				 efx_channel_num_tx_queues(_channel);		\
15161558 		     _tx_queue++)
15171559 
15181560 static inline bool efx_channel_has_rx_queue(struct efx_channel *channel)
....@@ -1556,6 +1598,15 @@
15561598 	return &rx_queue->buffer[index];
15571599 }
15581600 
1601
+static inline struct efx_rx_buffer *
1602
+efx_rx_buf_next(struct efx_rx_queue *rx_queue, struct efx_rx_buffer *rx_buf)
1603
+{
1604
+	if (unlikely(rx_buf == efx_rx_buffer(rx_queue, rx_queue->ptr_mask)))
1605
+		return efx_rx_buffer(rx_queue, 0);
1606
+	else
1607
+		return rx_buf + 1;
1608
+}
1609
+
15591610 /**
15601611  * EFX_MAX_FRAME_LEN - calculate maximum frame length
15611612  *
....@@ -1586,6 +1637,34 @@
15861637 	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
15871638 }
15881639 
1640
+/* Get the max fill level of the TX queues on this channel */
1641
+static inline unsigned int
1642
+efx_channel_tx_fill_level(struct efx_channel *channel)
1643
+{
1644
+	struct efx_tx_queue *tx_queue;
1645
+	unsigned int fill_level = 0;
1646
+
1647
+	efx_for_each_channel_tx_queue(tx_queue, channel)
1648
+		fill_level = max(fill_level,
1649
+				 tx_queue->insert_count - tx_queue->read_count);
1650
+
1651
+	return fill_level;
1652
+}
1653
+
1654
+/* Conservative approximation of efx_channel_tx_fill_level using cached value */
1655
+static inline unsigned int
1656
+efx_channel_tx_old_fill_level(struct efx_channel *channel)
1657
+{
1658
+	struct efx_tx_queue *tx_queue;
1659
+	unsigned int fill_level = 0;
1660
+
1661
+	efx_for_each_channel_tx_queue(tx_queue, channel)
1662
+		fill_level = max(fill_level,
1663
+				 tx_queue->insert_count - tx_queue->old_read_count);
1664
+
1665
+	return fill_level;
1666
+}
1667
+
15891668 /* Get all supported features.
15901669  * If a feature is not fixed, it is present in hw_features.
15911670  * If a feature is fixed, it does not present in hw_features, but