forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc 
 kernel/drivers/net/ethernet/sfc/net_driver.h
....@@ -1,11 +1,8 @@
1
+/* SPDX-License-Identifier: GPL-2.0-only */
12 /****************************************************************************
23  * Driver for Solarflare network controllers and boards
34  * Copyright 2005-2006 Fen Systems Ltd.
45  * Copyright 2005-2013 Solarflare Communications Inc.
5
- *
6
- * This program is free software; you can redistribute it and/or modify it
7
- * under the terms of the GNU General Public License version 2 as published
8
- * by the Free Software Foundation, incorporated herein by reference.
96  */
107 
118 /* Common definitions for all Efx net driver code */
....@@ -27,9 +24,9 @@
2724 #include <linux/mutex.h>
2825 #include <linux/rwsem.h>
2926 #include <linux/vmalloc.h>
30
-#include <linux/i2c.h>
3127 #include <linux/mtd/mtd.h>
3228 #include <net/busy_poll.h>
29
+#include <net/xdp.h>
3330 
3431 #include "enum.h"
3532 #include "bitfield.h"
....@@ -40,8 +37,6 @@
4037  * Build definitions
4138  *
4239  **************************************************************************/
43
-
44
-#define EFX_DRIVER_VERSION	"4.1"
4540 
4641 #ifdef DEBUG
4742 #define EFX_WARN_ON_ONCE_PARANOID(x) WARN_ON_ONCE(x)
....@@ -68,16 +63,22 @@
6863  * queues. */
6964 #define EFX_MAX_TX_TC		2
7065 #define EFX_MAX_CORE_TX_QUEUES	(EFX_MAX_TX_TC * EFX_MAX_CHANNELS)
71
-#define EFX_TXQ_TYPE_OFFLOAD	1	/* flag */
72
-#define EFX_TXQ_TYPE_HIGHPRI	2	/* flag */
73
-#define EFX_TXQ_TYPES		4
74
-#define EFX_MAX_TX_QUEUES	(EFX_TXQ_TYPES * EFX_MAX_CHANNELS)
66
+#define EFX_TXQ_TYPE_OUTER_CSUM	1	/* Outer checksum offload */
67
+#define EFX_TXQ_TYPE_INNER_CSUM	2	/* Inner checksum offload */
68
+#define EFX_TXQ_TYPE_HIGHPRI	4	/* High-priority (for TC) */
69
+#define EFX_TXQ_TYPES		8
70
+/* HIGHPRI is Siena-only, and INNER_CSUM is EF10, so no need for both */
71
+#define EFX_MAX_TXQ_PER_CHANNEL	4
72
+#define EFX_MAX_TX_QUEUES	(EFX_MAX_TXQ_PER_CHANNEL * EFX_MAX_CHANNELS)
7573 
7674 /* Maximum possible MTU the driver supports */
7775 #define EFX_MAX_MTU (9 * 1024)
7876 
7977 /* Minimum MTU, from RFC791 (IP) */
8078 #define EFX_MIN_MTU 68
79
+
80
+/* Maximum total header length for TSOv2 */
81
+#define EFX_TSO2_MAX_HDRLEN	208
8182 
8283 /* Size of an RX scatter buffer.  Small enough to pack 2 into a 4K page,
8384  * and should be a multiple of the cache line size.
....@@ -93,6 +94,12 @@
9394 #else
9495 #define EFX_RX_BUF_ALIGNMENT	4
9596 #endif
97
+
98
+/* Non-standard XDP_PACKET_HEADROOM and tailroom to satisfy XDP_REDIRECT and
99
+ * still fit two standard MTU size packets into a single 4K page.
100
+ */
101
+#define EFX_XDP_HEADROOM	128
102
+#define EFX_XDP_TAILROOM	SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
96103 
97104 /* Forward declare Precision Time Protocol (PTP) support structure. */
98105 struct efx_ptp_data;
....@@ -139,7 +146,10 @@
139146  * struct efx_tx_buffer - buffer state for a TX descriptor
140147  * @skb: When @flags & %EFX_TX_BUF_SKB, the associated socket buffer to be
141148  *	freed when descriptor completes
142
- * @option: When @flags & %EFX_TX_BUF_OPTION, a NIC-specific option descriptor.
149
+ * @xdpf: When @flags & %EFX_TX_BUF_XDP, the XDP frame information; its @data
150
+ *	member is the associated buffer to drop a page reference on.
151
+ * @option: When @flags & %EFX_TX_BUF_OPTION, an EF10-specific option
152
+ *	descriptor.
143153  * @dma_addr: DMA address of the fragment.
144154  * @flags: Flags for allocation and DMA mapping type
145155  * @len: Length of this fragment.
....@@ -149,9 +159,12 @@
149159  * Only valid if @unmap_len != 0.
150160  */
151161 struct efx_tx_buffer {
152
-	const struct sk_buff *skb;
153162 	union {
154
-		efx_qword_t option;
163
+		const struct sk_buff *skb;
164
+		struct xdp_frame *xdpf;
165
+	};
166
+	union {
167
+		efx_qword_t option;    /* EF10 */
155168 		dma_addr_t dma_addr;
156169 	};
157170 	unsigned short flags;
....@@ -163,6 +176,8 @@
163176 #define EFX_TX_BUF_SKB		2	/* buffer is last part of skb */
164177 #define EFX_TX_BUF_MAP_SINGLE	8	/* buffer was mapped with dma_map_single() */
165178 #define EFX_TX_BUF_OPTION	0x10	/* empty buffer for option descriptor */
179
+#define EFX_TX_BUF_XDP		0x20	/* buffer was sent with XDP */
180
+#define EFX_TX_BUF_TSO_V3	0x40	/* empty buffer for a TSO_V3 descriptor */
166181 
167182 /**
168183  * struct efx_tx_queue - An Efx TX queue
....@@ -179,7 +194,11 @@
179194  *
180195  * @efx: The associated Efx NIC
181196  * @queue: DMA queue number
197
+ * @label: Label for TX completion events.
198
+ *	Is our index within @channel->tx_queue array.
199
+ * @type: configuration type of this TX queue.  A bitmask of %EFX_TXQ_TYPE_* flags.
182200  * @tso_version: Version of TSO in use for this queue.
201
+ * @tso_encap: Is encapsulated TSO supported? Supported in TSOv2 on 8000 series.
183202  * @channel: The associated channel
184203  * @core_txq: The networking core TX queue structure
185204  * @buffer: The software buffer ring
....@@ -192,8 +211,7 @@
192211  * @piobuf_offset: Buffer offset to be specified in PIO descriptors
193212  * @initialised: Has hardware queue been initialised?
194213  * @timestamping: Is timestamping enabled for this channel?
195
- * @handle_tso: TSO xmit preparation handler.  Sets up the TSO metadata and
196
- *	may also map tx data, depending on the nature of the TSO implementation.
214
+ * @xdp_tx: Is this an XDP tx queue?
197215  * @read_count: Current read pointer.
198216  *	This is the number of buffers that have been removed from both rings.
199217  * @old_write_count: The value of @write_count when last checked.
....@@ -203,8 +221,6 @@
203221  *	avoid cache-line ping-pong between the xmit path and the
204222  *	completion path.
205223  * @merge_events: Number of TX merged completion events
206
- * @completed_desc_ptr: Most recent completed pointer - only used with
207
- *      timestamping.
208224  * @completed_timestamp_major: Top part of the most recent tx timestamp.
209225  * @completed_timestamp_minor: Low part of the most recent tx timestamp.
210226  * @insert_count: Current insert pointer
....@@ -232,8 +248,9 @@
232248  * @tso_fallbacks: Number of times TSO fallback used
233249  * @pushes: Number of times the TX push feature has been used
234250  * @pio_packets: Number of times the TX PIO feature has been used
235
- * @xmit_more_available: Are any packets waiting to be pushed to the NIC
251
+ * @xmit_pending: Are any packets waiting to be pushed to the NIC
236252  * @cb_packets: Number of times the TX copybreak feature has been used
253
+ * @notify_count: Count of notified descriptors to the NIC
237254  * @empty_read_count: If the completion path has seen the queue as empty
238255  *	and the transmission path has not yet checked this, the value of
239256  *	@read_count bitwise-added to %EFX_EMPTY_COUNT_VALID; otherwise 0.
....@@ -241,8 +258,11 @@
241258 struct efx_tx_queue {
242259 	/* Members which don't change on the fast path */
243260 	struct efx_nic *efx ____cacheline_aligned_in_smp;
244
-	unsigned queue;
261
+	unsigned int queue;
262
+	unsigned int label;
263
+	unsigned int type;
245264 	unsigned int tso_version;
265
+	bool tso_encap;
246266 	struct efx_channel *channel;
247267 	struct netdev_queue *core_txq;
248268 	struct efx_tx_buffer *buffer;
....@@ -253,9 +273,7 @@
253273 	unsigned int piobuf_offset;
254274 	bool initialised;
255275 	bool timestamping;
256
-
257
-	/* Function pointers used in the fast path. */
258
-	int (*handle_tso)(struct efx_tx_queue*, struct sk_buff*, bool *);
276
+	bool xdp_tx;
259277 
260278 	/* Members used mainly on the completion path */
261279 	unsigned int read_count ____cacheline_aligned_in_smp;
....@@ -263,7 +281,6 @@
263281 	unsigned int merge_events;
264282 	unsigned int bytes_compl;
265283 	unsigned int pkts_compl;
266
-	unsigned int completed_desc_ptr;
267284 	u32 completed_timestamp_major;
268285 	u32 completed_timestamp_minor;
269286 
....@@ -278,8 +295,9 @@
278295 	unsigned int tso_fallbacks;
279296 	unsigned int pushes;
280297 	unsigned int pio_packets;
281
-	bool xmit_more_available;
298
+	bool xmit_pending;
282299 	unsigned int cb_packets;
300
+	unsigned int notify_count;
283301 	/* Statistics to supplement MAC stats */
284302 	unsigned long tx_packets;
285303 
....@@ -330,7 +348,7 @@
330348 struct efx_rx_page_state {
331349 	dma_addr_t dma_addr;
332350 
333
-	unsigned int __pad[0] ____cacheline_aligned;
351
+	unsigned int __pad[] ____cacheline_aligned;
334352 };
335353 
336354 /**
....@@ -366,6 +384,8 @@
366384  *	refill was triggered.
367385  * @recycle_count: RX buffer recycle counter.
368386  * @slow_fill: Timer used to defer efx_nic_generate_fill_event().
387
+ * @xdp_rxq_info: XDP specific RX queue information.
388
+ * @xdp_rxq_info_valid: Is xdp_rxq_info valid data?.
369389  */
370390 struct efx_rx_queue {
371391 	struct efx_nic *efx;
....@@ -397,6 +417,8 @@
397417 	unsigned int slow_fill_count;
398418 	/* Statistics to supplement MAC stats */
399419 	unsigned long rx_packets;
420
+	struct xdp_rxq_info xdp_rxq_info;
421
+	bool xdp_rxq_info_valid;
400422 };
401423 
402424 enum efx_sync_events_state {
....@@ -430,6 +452,13 @@
430452  * @event_test_cpu: Last CPU to handle interrupt or test event for this channel
431453  * @irq_count: Number of IRQs since last adaptive moderation decision
432454  * @irq_mod_score: IRQ moderation score
455
+ * @rfs_filter_count: number of accelerated RFS filters currently in place;
456
+ *	equals the count of @rps_flow_id slots filled
457
+ * @rfs_last_expiry: value of jiffies last time some accelerated RFS filters
458
+ *	were checked for expiry
459
+ * @rfs_expire_index: next accelerated RFS filter ID to check for expiry
460
+ * @n_rfs_succeeded: number of successful accelerated RFS filter insertions
461
+ * @n_rfs_failed: number of failed accelerated RFS filter insertions
433462  * @filter_work: Work item for efx_filter_rfs_expire()
434463  * @rps_flow_id: Flow IDs of filters allocated for accelerated RFS,
435464  *      indexed by filter ID
....@@ -444,6 +473,10 @@
444473  *	lack of descriptors
445474  * @n_rx_merge_events: Number of RX merged completion events
446475  * @n_rx_merge_packets: Number of RX packets completed by merged events
476
+ * @n_rx_xdp_drops: Count of RX packets intentionally dropped due to XDP
477
+ * @n_rx_xdp_bad_drops: Count of RX packets dropped due to XDP errors
478
+ * @n_rx_xdp_tx: Count of RX packets retransmitted due to XDP
479
+ * @n_rx_xdp_redirect: Count of RX packets redirected to a different NIC by XDP
447480  * @rx_pkt_n_frags: Number of fragments in next packet to be delivered by
448481  *	__efx_rx_packet(), or zero if there is none
449482  * @rx_pkt_index: Ring index of first buffer for next packet to be delivered
....@@ -451,6 +484,7 @@
451484  * @rx_list: list of SKBs from current RX, awaiting processing
452485  * @rx_queue: RX queue for this channel
453486  * @tx_queue: TX queues for this channel
487
+ * @tx_queue_by_type: pointers into @tx_queue, or %NULL, indexed by txq type
454488  * @sync_events_state: Current state of sync events on this channel
455489  * @sync_timestamp_major: Major part of the last ptp sync event
456490  * @sync_timestamp_minor: Minor part of the last ptp sync event
....@@ -476,8 +510,12 @@
476510 	unsigned int irq_count;
477511 	unsigned int irq_mod_score;
478512 #ifdef CONFIG_RFS_ACCEL
479
-	unsigned int rfs_filters_added;
480
-	struct work_struct filter_work;
513
+	unsigned int rfs_filter_count;
514
+	unsigned int rfs_last_expiry;
515
+	unsigned int rfs_expire_index;
516
+	unsigned int n_rfs_succeeded;
517
+	unsigned int n_rfs_failed;
518
+	struct delayed_work filter_work;
481519 #define RPS_FLOW_ID_INVALID 0xFFFFFFFF
482520 	u32 *rps_flow_id;
483521 #endif
....@@ -497,6 +535,10 @@
497535 	unsigned int n_rx_nodesc_trunc;
498536 	unsigned int n_rx_merge_events;
499537 	unsigned int n_rx_merge_packets;
538
+	unsigned int n_rx_xdp_drops;
539
+	unsigned int n_rx_xdp_bad_drops;
540
+	unsigned int n_rx_xdp_tx;
541
+	unsigned int n_rx_xdp_redirect;
500542 
501543 	unsigned int rx_pkt_n_frags;
502544 	unsigned int rx_pkt_index;
....@@ -504,7 +546,8 @@
504546 	struct list_head *rx_list;
505547 
506548 	struct efx_rx_queue rx_queue;
507
-	struct efx_tx_queue tx_queue[EFX_TXQ_TYPES];
549
+	struct efx_tx_queue tx_queue[EFX_MAX_TXQ_PER_CHANNEL];
550
+	struct efx_tx_queue *tx_queue_by_type[EFX_TXQ_TYPES];
508551 
509552 	enum efx_sync_events_state sync_events_state;
510553 	u32 sync_timestamp_major;
....@@ -574,8 +617,6 @@
574617 #define RESET_TYPE(type) \
575618 	STRING_TABLE_LOOKUP(type, efx_reset_type)
576619 
577
-void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen);
578
-
579620 enum efx_int_mode {
580621 	/* Be careful if altering to correct macro below */
581622 	EFX_INT_MODE_MSIX = 0,
....@@ -586,11 +627,53 @@
586627 #define EFX_INT_MODE_USE_MSI(x) (((x)->interrupt_mode) <= EFX_INT_MODE_MSI)
587628 
588629 enum nic_state {
589
-	STATE_UNINIT = 0,	/* device being probed/removed or is frozen */
590
-	STATE_READY = 1,	/* hardware ready and netdev registered */
591
-	STATE_DISABLED = 2,	/* device disabled due to hardware errors */
592
-	STATE_RECOVERY = 3,	/* device recovering from PCI error */
630
+	STATE_UNINIT = 0,	/* device being probed/removed */
631
+	STATE_NET_DOWN,		/* hardware probed and netdev registered */
632
+	STATE_NET_UP,		/* ready for traffic */
633
+	STATE_DISABLED,		/* device disabled due to hardware errors */
634
+
635
+	STATE_RECOVERY = 0x100,/* recovering from PCI error */
636
+	STATE_FROZEN = 0x200,	/* frozen by power management */
593637 };
638
+
639
+static inline bool efx_net_active(enum nic_state state)
640
+{
641
+	return state == STATE_NET_DOWN || state == STATE_NET_UP;
642
+}
643
+
644
+static inline bool efx_frozen(enum nic_state state)
645
+{
646
+	return state & STATE_FROZEN;
647
+}
648
+
649
+static inline bool efx_recovering(enum nic_state state)
650
+{
651
+	return state & STATE_RECOVERY;
652
+}
653
+
654
+static inline enum nic_state efx_freeze(enum nic_state state)
655
+{
656
+	WARN_ON(!efx_net_active(state));
657
+	return state | STATE_FROZEN;
658
+}
659
+
660
+static inline enum nic_state efx_thaw(enum nic_state state)
661
+{
662
+	WARN_ON(!efx_frozen(state));
663
+	return state & ~STATE_FROZEN;
664
+}
665
+
666
+static inline enum nic_state efx_recover(enum nic_state state)
667
+{
668
+	WARN_ON(!efx_net_active(state));
669
+	return state | STATE_RECOVERY;
670
+}
671
+
672
+static inline enum nic_state efx_recovered(enum nic_state state)
673
+{
674
+	WARN_ON(!efx_recovering(state));
675
+	return state & ~STATE_RECOVERY;
676
+}
594677 
595678 /* Forward declaration */
596679 struct efx_nic;
....@@ -620,51 +703,6 @@
620703 	return left->up == right->up && left->fd == right->fd &&
621704 		left->fc == right->fc && left->speed == right->speed;
622705 }
623
-
624
-/**
625
- * struct efx_phy_operations - Efx PHY operations table
626
- * @probe: Probe PHY and initialise efx->mdio.mode_support, efx->mdio.mmds,
627
- *	efx->loopback_modes.
628
- * @init: Initialise PHY
629
- * @fini: Shut down PHY
630
- * @reconfigure: Reconfigure PHY (e.g. for new link parameters)
631
- * @poll: Update @link_state and report whether it changed.
632
- *	Serialised by the mac_lock.
633
- * @get_link_ksettings: Get ethtool settings. Serialised by the mac_lock.
634
- * @set_link_ksettings: Set ethtool settings. Serialised by the mac_lock.
635
- * @get_fecparam: Get Forward Error Correction settings. Serialised by mac_lock.
636
- * @set_fecparam: Set Forward Error Correction settings. Serialised by mac_lock.
637
- * @set_npage_adv: Set abilities advertised in (Extended) Next Page
638
- *	(only needed where AN bit is set in mmds)
639
- * @test_alive: Test that PHY is 'alive' (online)
640
- * @test_name: Get the name of a PHY-specific test/result
641
- * @run_tests: Run tests and record results as appropriate (offline).
642
- *	Flags are the ethtool tests flags.
643
- */
644
-struct efx_phy_operations {
645
-	int (*probe) (struct efx_nic *efx);
646
-	int (*init) (struct efx_nic *efx);
647
-	void (*fini) (struct efx_nic *efx);
648
-	void (*remove) (struct efx_nic *efx);
649
-	int (*reconfigure) (struct efx_nic *efx);
650
-	bool (*poll) (struct efx_nic *efx);
651
-	void (*get_link_ksettings)(struct efx_nic *efx,
652
-				   struct ethtool_link_ksettings *cmd);
653
-	int (*set_link_ksettings)(struct efx_nic *efx,
654
-				  const struct ethtool_link_ksettings *cmd);
655
-	int (*get_fecparam)(struct efx_nic *efx, struct ethtool_fecparam *fec);
656
-	int (*set_fecparam)(struct efx_nic *efx,
657
-			    const struct ethtool_fecparam *fec);
658
-	void (*set_npage_adv) (struct efx_nic *efx, u32);
659
-	int (*test_alive) (struct efx_nic *efx);
660
-	const char *(*test_name) (struct efx_nic *efx, unsigned int index);
661
-	int (*run_tests) (struct efx_nic *efx, int *results, unsigned flags);
662
-	int (*get_module_eeprom) (struct efx_nic *efx,
663
-			       struct ethtool_eeprom *ee,
664
-			       u8 *data);
665
-	int (*get_module_info) (struct efx_nic *efx,
666
-				struct ethtool_modinfo *modinfo);
667
-};
668706 
669707 /**
670708  * enum efx_phy_mode - PHY operating mode flags
....@@ -715,13 +753,13 @@
715753 struct vfdi_status;
716754 
717755 /* The reserved RSS context value */
718
-#define EFX_EF10_RSS_CONTEXT_INVALID	0xffffffff
756
+#define EFX_MCDI_RSS_CONTEXT_INVALID	0xffffffff
719757 /**
720758  * struct efx_rss_context - A user-defined RSS context for filtering
721759  * @list: node of linked list on which this struct is stored
722760  * @context_id: the RSS_CONTEXT_ID returned by MC firmware, or
723
- *	%EFX_EF10_RSS_CONTEXT_INVALID if this context is not present on the NIC.
724
- *	For Siena, 0 if RSS is active, else %EFX_EF10_RSS_CONTEXT_INVALID.
761
+ *	%EFX_MCDI_RSS_CONTEXT_INVALID if this context is not present on the NIC.
762
+ *	For Siena, 0 if RSS is active, else %EFX_MCDI_RSS_CONTEXT_INVALID.
725763  * @user_id: the rss_context ID exposed to userspace over ethtool.
726764  * @rx_hash_udp_4tuple: UDP 4-tuple hashing enabled
727765  * @rx_hash_key: Toeplitz hash key for this RSS context
....@@ -810,6 +848,7 @@
810848  * @timer_quantum_ns: Interrupt timer quantum, in nanoseconds
811849  * @timer_max_ns: Interrupt timer maximum value, in nanoseconds
812850  * @irq_rx_adaptive: Adaptive IRQ moderation enabled for RX event queues
851
+ * @irqs_hooked: Channel interrupts are hooked
813852  * @irq_rx_mod_step_us: Step size for IRQ moderation for RX event queues
814853  * @irq_rx_moderation_us: IRQ moderation time for RX event queues
815854  * @msg_enable: Log message enable flags
....@@ -821,6 +860,8 @@
821860  * @msi_context: Context for each MSI
822861  * @extra_channel_types: Types of extra (non-traffic) channels that
823862  *	should be allocated for this NIC
863
+ * @xdp_tx_queue_count: Number of entries in %xdp_tx_queues.
864
+ * @xdp_tx_queues: Array of pointers to tx queues used for XDP transmit.
824865  * @rxq_entries: Size of receive queues requested by user.
825866  * @txq_entries: Size of transmit queues requested by user.
826867  * @txq_stop_thresh: TX queue fill level at or above which we stop it.
....@@ -833,6 +874,10 @@
833874  * @n_rx_channels: Number of channels used for RX (= number of RX queues)
834875  * @n_tx_channels: Number of channels used for TX
835876  * @n_extra_tx_channels: Number of extra channels with TX queues
877
+ * @tx_queues_per_channel: number of TX queues probed on each channel
878
+ * @n_xdp_channels: Number of channels used for XDP TX
879
+ * @xdp_channel_offset: Offset of zeroth channel used for XPD TX.
880
+ * @xdp_tx_per_channel: Max number of TX queues on an XDP TX channel.
836881  * @rx_ip_align: RX DMA address offset to have IP header aligned in
837882  *	in accordance with NET_IP_ALIGN
838883  * @rx_dma_len: Current maximum RX DMA length
....@@ -850,8 +895,10 @@
850895  * @rss_context: Main RSS context.  Its @list member is the head of the list of
851896  *	RSS contexts created by user requests
852897  * @rss_lock: Protects custom RSS context software state in @rss_context.list
898
+ * @vport_id: The function's vport ID, only relevant for PFs
853899  * @int_error_count: Number of internal errors seen recently
854900  * @int_error_expire: Time at which error count will be expired
901
+ * @must_realloc_vis: Flag: VIs have yet to be reallocated after MC reboot
855902  * @irq_soft_enabled: Are IRQs soft-enabled? If not, IRQ handler will
856903  *	acknowledge but do nothing else.
857904  * @irq_status: Interrupt status buffer
....@@ -875,7 +922,6 @@
875922  *	field of %MC_CMD_GET_CAPABILITIES_V4 response, or %MC_CMD_MAC_NSTATS)
876923  * @stats_buffer: DMA buffer for statistics
877924  * @phy_type: PHY type
878
- * @phy_op: PHY interface
879925  * @phy_data: PHY private data (including PHY-specific stats)
880926  * @mdio: PHY MDIO interface
881927  * @mdio_bus: PHY MDIO bus ID (only used by Siena)
....@@ -897,12 +943,10 @@
897943  * @loopback_mode: Loopback status
898944  * @loopback_modes: Supported loopback mode bitmask
899945  * @loopback_selftest: Offline self-test private state
946
+ * @xdp_prog: Current XDP programme for this interface
900947  * @filter_sem: Filter table rw_semaphore, protects existence of @filter_state
901948  * @filter_state: Architecture-dependent filter table state
902949  * @rps_mutex: Protects RPS state of all channels
903
- * @rps_expire_channel: Next channel to check for expiry
904
- * @rps_expire_index: Next index to check for expiry in
905
- *	@rps_expire_channel's @rps_flow_id
906950  * @rps_slot_map: bitmap of in-flight entries in @rps_slot
907951  * @rps_slot: array of ARFS insertion requests for efx_filter_rfs_work()
908952  * @rps_hash_lock: Protects ARFS filter mapping state (@rps_hash_table and
....@@ -922,6 +966,11 @@
922966  * @ptp_data: PTP state data
923967  * @ptp_warned: has this NIC seen and warned about unexpected PTP events?
924968  * @vpd_sn: Serial number read from VPD
969
+ * @xdp_rxq_info_failed: Have any of the rx queues failed to initialise their
970
+ *      xdp_rxq_info structures?
971
+ * @netdev_notifier: Netdevice notifier.
972
+ * @mem_bar: The BAR that is mapped into membase.
973
+ * @reg_base: Offset from the start of the bar to the function control window.
925974  * @monitor_work: Hardware monitor workitem
926975  * @biu_lock: BIU (bus interface unit) lock
927976  * @last_irq_cpu: Last CPU to handle a possible test interrupt.  This
....@@ -957,6 +1006,7 @@
9571006 	unsigned int timer_quantum_ns;
9581007 	unsigned int timer_max_ns;
9591008 	bool irq_rx_adaptive;
1009
+	bool irqs_hooked;
9601010 	unsigned int irq_mod_step_us;
9611011 	unsigned int irq_rx_moderation_us;
9621012 	u32 msg_enable;
....@@ -969,6 +1019,9 @@
9691019 	const struct efx_channel_type *
9701020 	extra_channel_type[EFX_MAX_EXTRA_CHANNELS];
9711021 
1022
+	unsigned int xdp_tx_queue_count;
1023
+	struct efx_tx_queue **xdp_tx_queues;
1024
+
9721025 	unsigned rxq_entries;
9731026 	unsigned txq_entries;
9741027 	unsigned int txq_stop_thresh;
....@@ -980,6 +1033,7 @@
9801033 	unsigned next_buffer_table;
9811034 
9821035 	unsigned int max_channels;
1036
+	unsigned int max_vis;
9831037 	unsigned int max_tx_channels;
9841038 	unsigned n_channels;
9851039 	unsigned n_rx_channels;
....@@ -987,6 +1041,10 @@
9871041 	unsigned tx_channel_offset;
9881042 	unsigned n_tx_channels;
9891043 	unsigned n_extra_tx_channels;
1044
+	unsigned int tx_queues_per_channel;
1045
+	unsigned int n_xdp_channels;
1046
+	unsigned int xdp_channel_offset;
1047
+	unsigned int xdp_tx_per_channel;
9901048 	unsigned int rx_ip_align;
9911049 	unsigned int rx_dma_len;
9921050 	unsigned int rx_buffer_order;
....@@ -1001,10 +1059,12 @@
10011059 	bool rx_scatter;
10021060 	struct efx_rss_context rss_context;
10031061 	struct mutex rss_lock;
1062
+	u32 vport_id;
10041063 
10051064 	unsigned int_error_count;
10061065 	unsigned long int_error_expire;
10071066 
1067
+	bool must_realloc_vis;
10081068 	bool irq_soft_enabled;
10091069 	struct efx_buffer irq_status;
10101070 	unsigned irq_zero_count;
....@@ -1035,7 +1095,6 @@
10351095 	bool rx_nodesc_drops_prev_state;
10361096 
10371097 	unsigned int phy_type;
1038
-	const struct efx_phy_operations *phy_op;
10391098 	void *phy_data;
10401099 	struct mdio_if_info mdio;
10411100 	unsigned int mdio_bus;
....@@ -1056,13 +1115,15 @@
10561115 	u64 loopback_modes;
10571116 
10581117 	void *loopback_selftest;
1118
+	/* We access loopback_selftest immediately before running XDP,
1119
+	 * so we want them next to each other.
1120
+	 */
1121
+	struct bpf_prog __rcu *xdp_prog;
10591122 
10601123 	struct rw_semaphore filter_sem;
10611124 	void *filter_state;
10621125 #ifdef CONFIG_RFS_ACCEL
10631126 	struct mutex rps_mutex;
1064
-	unsigned int rps_expire_channel;
1065
-	unsigned int rps_expire_index;
10661127 	unsigned long rps_slot_map;
10671128 	struct efx_async_filter_insertion rps_slot[EFX_RPS_MAX_IN_FLIGHT];
10681129 	spinlock_t rps_hash_lock;
....@@ -1085,6 +1146,12 @@
10851146 	bool ptp_warned;
10861147 
10871148 	char *vpd_sn;
1149
+	bool xdp_rxq_info_failed;
1150
+
1151
+	struct notifier_block netdev_notifier;
1152
+
1153
+	unsigned int mem_bar;
1154
+	u32 reg_base;
10881155 
10891156 	/* The following fields may be written more often */
10901157 
....@@ -1114,12 +1181,9 @@
11141181 };
11151182 
11161183 struct efx_udp_tunnel {
1184
+#define TUNNEL_ENCAP_UDP_PORT_ENTRY_INVALID	0xffff
11171185 	u16 type; /* TUNNEL_ENCAP_UDP_PORT_ENTRY_foo, see mcdi_pcol.h */
11181186 	__be16 port;
1119
-	/* Count of repeated adds of the same port.  Used only inside the list,
1120
-	 * not in request arguments.
1121
-	 */
1122
-	u16 count;
11231187 };
11241188 
11251189 /**
....@@ -1150,10 +1214,12 @@
11501214  * @describe_stats: Describe statistics for ethtool
11511215  * @update_stats: Update statistics not provided by event handling.
11521216  *	Either argument may be %NULL.
1217
+ * @update_stats_atomic: Update statistics while in atomic context, if that
1218
+ *	is more limiting than @update_stats.  Otherwise, leave %NULL and
1219
+ *	driver core will call @update_stats.
11531220  * @start_stats: Start the regular fetching of statistics
11541221  * @pull_stats: Pull stats from the NIC and wait until they arrive.
11551222  * @stop_stats: Stop the regular fetching of statistics
1156
- * @set_id_led: Set state of identifying LED or revert to automatic function
11571223  * @push_irq_moderation: Apply interrupt moderation value
11581224  * @reconfigure_port: Push loopback/power/txdis changes to the MAC and PHY
11591225  * @prepare_enable_fc_tx: Prepare MAC to enable pause frame TX (may be %NULL)
....@@ -1186,10 +1252,11 @@
11861252  *	a pointer to the &struct efx_msi_context for the channel.
11871253  * @irq_handle_legacy: Handle legacy interrupt.  The @dev_id argument
11881254  *	is a pointer to the &struct efx_nic.
1189
- * @tx_probe: Allocate resources for TX queue
1255
+ * @tx_probe: Allocate resources for TX queue (and select TXQ type)
11901256  * @tx_init: Initialise TX queue on the NIC
11911257  * @tx_remove: Free resources for TX queue
11921258  * @tx_write: Write TX descriptors and doorbell
1259
+ * @tx_enqueue: Add an SKB to TX queue
11931260  * @rx_push_rss_config: Write RSS hash key and indirection table to the NIC
11941261  * @rx_pull_rss_config: Read RSS hash key and indirection table back from the NIC
11951262  * @rx_push_rss_context_config: Write RSS hash key and indirection table for
....@@ -1201,6 +1268,8 @@
12011268  * @rx_remove: Free resources for RX queue
12021269  * @rx_write: Write RX descriptors and doorbell
12031270  * @rx_defer_refill: Generate a refill reminder event
1271
+ * @rx_packet: Receive the queued RX buffer on a channel
1272
+ * @rx_buf_hash_valid: Determine whether the RX prefix contains a valid hash
12041273  * @ev_probe: Allocate resources for event queue
12051274  * @ev_init: Initialise event queue on the NIC
12061275  * @ev_fini: Deinitialise event queue on the NIC
....@@ -1243,9 +1312,9 @@
12431312  * @tso_versions: Returns mask of firmware-assisted TSO versions supported.
12441313  *	If %NULL, then device does not support any TSO version.
12451314  * @udp_tnl_push_ports: Push the list of UDP tunnel ports to the NIC if required.
1246
- * @udp_tnl_add_port: Add a UDP tunnel port
12471315  * @udp_tnl_has_port: Check if a port has been added as UDP tunnel
1248
- * @udp_tnl_del_port: Remove a UDP tunnel port
1316
+ * @print_additional_fwver: Dump NIC-specific additional FW version info
1317
+ * @sensor_event: Handle a sensor event from MCDI
12491318  * @revision: Hardware architecture revision
12501319  * @txd_ptr_tbl_base: TX descriptor ring base address
12511320  * @rxd_ptr_tbl_base: RX descriptor ring base address
....@@ -1261,8 +1330,6 @@
12611330  * @always_rx_scatter: NIC will always scatter packets to multiple buffers
12621331  * @option_descriptors: NIC supports TX option descriptors
12631332  * @min_interrupt_mode: Lowest capability interrupt mode supported
1264
- *	from &enum efx_int_mode.
1265
- * @max_interrupt_mode: Highest capability interrupt mode supported
12661333  *	from &enum efx_int_mode.
12671334  * @timer_period_max: Maximum period of interrupt timer (in ticks)
12681335  * @offload_features: net_device feature flags for protocol offload
....@@ -1294,18 +1361,22 @@
12941361 	size_t (*describe_stats)(struct efx_nic *efx, u8 *names);
12951362 	size_t (*update_stats)(struct efx_nic *efx, u64 *full_stats,
12961363 			       struct rtnl_link_stats64 *core_stats);
1364
+	size_t (*update_stats_atomic)(struct efx_nic *efx, u64 *full_stats,
1365
+				      struct rtnl_link_stats64 *core_stats);
12971366 	void (*start_stats)(struct efx_nic *efx);
12981367 	void (*pull_stats)(struct efx_nic *efx);
12991368 	void (*stop_stats)(struct efx_nic *efx);
1300
-	void (*set_id_led)(struct efx_nic *efx, enum efx_led_mode mode);
13011369 	void (*push_irq_moderation)(struct efx_channel *channel);
13021370 	int (*reconfigure_port)(struct efx_nic *efx);
13031371 	void (*prepare_enable_fc_tx)(struct efx_nic *efx);
1304
-	int (*reconfigure_mac)(struct efx_nic *efx);
1372
+	int (*reconfigure_mac)(struct efx_nic *efx, bool mtu_only);
13051373 	bool (*check_mac_fault)(struct efx_nic *efx);
13061374 	void (*get_wol)(struct efx_nic *efx, struct ethtool_wolinfo *wol);
13071375 	int (*set_wol)(struct efx_nic *efx, u32 type);
13081376 	void (*resume_wol)(struct efx_nic *efx);
1377
+	unsigned int (*check_caps)(const struct efx_nic *efx,
1378
+				   u8 flag,
1379
+				   u32 offset);
13091380 	int (*test_chip)(struct efx_nic *efx, struct efx_self_tests *tests);
13101381 	int (*test_nvram)(struct efx_nic *efx);
13111382 	void (*mcdi_request)(struct efx_nic *efx,
....@@ -1325,6 +1396,7 @@
13251396 	void (*tx_init)(struct efx_tx_queue *tx_queue);
13261397 	void (*tx_remove)(struct efx_tx_queue *tx_queue);
13271398 	void (*tx_write)(struct efx_tx_queue *tx_queue);
1399
+	netdev_tx_t (*tx_enqueue)(struct efx_tx_queue *tx_queue, struct sk_buff *skb);
13281400 	unsigned int (*tx_limit_len)(struct efx_tx_queue *tx_queue,
13291401 				     dma_addr_t dma_addr, unsigned int len);
13301402 	int (*rx_push_rss_config)(struct efx_nic *efx, bool user,
....@@ -1342,6 +1414,8 @@
13421414 	void (*rx_remove)(struct efx_rx_queue *rx_queue);
13431415 	void (*rx_write)(struct efx_rx_queue *rx_queue);
13441416 	void (*rx_defer_refill)(struct efx_rx_queue *rx_queue);
1417
+	void (*rx_packet)(struct efx_channel *channel);
1418
+	bool (*rx_buf_hash_valid)(const u8 *prefix);
13451419 	int (*ev_probe)(struct efx_channel *channel);
13461420 	int (*ev_init)(struct efx_channel *channel);
13471421 	void (*ev_fini)(struct efx_channel *channel);
....@@ -1413,9 +1487,10 @@
14131487 	int (*set_mac_address)(struct efx_nic *efx);
14141488 	u32 (*tso_versions)(struct efx_nic *efx);
14151489 	int (*udp_tnl_push_ports)(struct efx_nic *efx);
1416
-	int (*udp_tnl_add_port)(struct efx_nic *efx, struct efx_udp_tunnel tnl);
14171490 	bool (*udp_tnl_has_port)(struct efx_nic *efx, __be16 port);
1418
-	int (*udp_tnl_del_port)(struct efx_nic *efx, struct efx_udp_tunnel tnl);
1491
+	size_t (*print_additional_fwver)(struct efx_nic *efx, char *buf,
1492
+					 size_t len);
1493
+	void (*sensor_event)(struct efx_nic *efx, efx_qword_t *ev);
14191494 
14201495 	int revision;
14211496 	unsigned int txd_ptr_tbl_base;
....@@ -1432,7 +1507,6 @@
14321507 	bool always_rx_scatter;
14331508 	bool option_descriptors;
14341509 	unsigned int min_interrupt_mode;
1435
-	unsigned int max_interrupt_mode;
14361510 	unsigned int timer_period_max;
14371511 	netdev_features_t offload_features;
14381512 	int mcdi_max_ver;
....@@ -1468,32 +1542,51 @@
14681542 	     _channel = _channel->channel ?				\
14691543 		     (_efx)->channel[_channel->channel - 1] : NULL)
14701544 
1471
-static inline struct efx_tx_queue *
1472
-efx_get_tx_queue(struct efx_nic *efx, unsigned index, unsigned type)
1545
+static inline struct efx_channel *
1546
+efx_get_tx_channel(struct efx_nic *efx, unsigned int index)
14731547 {
1474
-	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_tx_channels ||
1475
-				  type >= EFX_TXQ_TYPES);
1476
-	return &efx->channel[efx->tx_channel_offset + index]->tx_queue[type];
1548
+	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_tx_channels);
1549
+	return efx->channel[efx->tx_channel_offset + index];
1550
+}
1551
+
1552
+static inline struct efx_channel *
1553
+efx_get_xdp_channel(struct efx_nic *efx, unsigned int index)
1554
+{
1555
+	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_xdp_channels);
1556
+	return efx->channel[efx->xdp_channel_offset + index];
1557
+}
1558
+
1559
+static inline bool efx_channel_is_xdp_tx(struct efx_channel *channel)
1560
+{
1561
+	return channel->channel - channel->efx->xdp_channel_offset <
1562
+	       channel->efx->n_xdp_channels;
14771563 }
14781564 
14791565 static inline bool efx_channel_has_tx_queues(struct efx_channel *channel)
14801566 {
1481
-	return channel->type && channel->type->want_txqs &&
1482
-				channel->type->want_txqs(channel);
1567
+	return channel && channel->channel >= channel->efx->tx_channel_offset;
1568
+}
1569
+
1570
+static inline unsigned int efx_channel_num_tx_queues(struct efx_channel *channel)
1571
+{
1572
+	if (efx_channel_is_xdp_tx(channel))
1573
+		return channel->efx->xdp_tx_per_channel;
1574
+	return channel->efx->tx_queues_per_channel;
14831575 }
14841576 
14851577 static inline struct efx_tx_queue *
1486
-efx_channel_get_tx_queue(struct efx_channel *channel, unsigned type)
1578
+efx_channel_get_tx_queue(struct efx_channel *channel, unsigned int type)
14871579 {
1488
-	EFX_WARN_ON_ONCE_PARANOID(!efx_channel_has_tx_queues(channel) ||
1489
-				  type >= EFX_TXQ_TYPES);
1490
-	return &channel->tx_queue[type];
1580
+	EFX_WARN_ON_ONCE_PARANOID(type >= EFX_TXQ_TYPES);
1581
+	return channel->tx_queue_by_type[type];
14911582 }
14921583 
1493
-static inline bool efx_tx_queue_used(struct efx_tx_queue *tx_queue)
1584
+static inline struct efx_tx_queue *
1585
+efx_get_tx_queue(struct efx_nic *efx, unsigned int index, unsigned int type)
14941586 {
1495
-	return !(tx_queue->efx->net_dev->num_tc < 2 &&
1496
-		 tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI);
1587
+	struct efx_channel *channel = efx_get_tx_channel(efx, index);
1588
+
1589
+	return efx_channel_get_tx_queue(channel, type);
14971590 }
14981591 
14991592 /* Iterate over all TX queues belonging to a channel */
....@@ -1502,17 +1595,8 @@
15021595 		;							\
15031596 	else								\
15041597 		for (_tx_queue = (_channel)->tx_queue;			\
1505
-		     _tx_queue < (_channel)->tx_queue + EFX_TXQ_TYPES && \
1506
-			     efx_tx_queue_used(_tx_queue);		\
1507
-		     _tx_queue++)
1508
-
1509
-/* Iterate over all possible TX queues belonging to a channel */
1510
-#define efx_for_each_possible_channel_tx_queue(_tx_queue, _channel)	\
1511
-	if (!efx_channel_has_tx_queues(_channel))			\
1512
-		;							\
1513
-	else								\
1514
-		for (_tx_queue = (_channel)->tx_queue;			\
1515
-		     _tx_queue < (_channel)->tx_queue + EFX_TXQ_TYPES;	\
1598
+		     _tx_queue < (_channel)->tx_queue +			\
1599
+				 efx_channel_num_tx_queues(_channel);		\
15161600 		     _tx_queue++)
15171601 
15181602 static inline bool efx_channel_has_rx_queue(struct efx_channel *channel)
....@@ -1556,6 +1640,15 @@
15561640 	return &rx_queue->buffer[index];
15571641 }
15581642 
1643
+static inline struct efx_rx_buffer *
1644
+efx_rx_buf_next(struct efx_rx_queue *rx_queue, struct efx_rx_buffer *rx_buf)
1645
+{
1646
+	if (unlikely(rx_buf == efx_rx_buffer(rx_queue, rx_queue->ptr_mask)))
1647
+		return efx_rx_buffer(rx_queue, 0);
1648
+	else
1649
+		return rx_buf + 1;
1650
+}
1651
+
15591652 /**
15601653  * EFX_MAX_FRAME_LEN - calculate maximum frame length
15611654  *
....@@ -1586,6 +1679,34 @@
15861679 	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
15871680 }
15881681 
1682
+/* Get the max fill level of the TX queues on this channel */
1683
+static inline unsigned int
1684
+efx_channel_tx_fill_level(struct efx_channel *channel)
1685
+{
1686
+	struct efx_tx_queue *tx_queue;
1687
+	unsigned int fill_level = 0;
1688
+
1689
+	efx_for_each_channel_tx_queue(tx_queue, channel)
1690
+		fill_level = max(fill_level,
1691
+				 tx_queue->insert_count - tx_queue->read_count);
1692
+
1693
+	return fill_level;
1694
+}
1695
+
1696
+/* Conservative approximation of efx_channel_tx_fill_level using cached value */
1697
+static inline unsigned int
1698
+efx_channel_tx_old_fill_level(struct efx_channel *channel)
1699
+{
1700
+	struct efx_tx_queue *tx_queue;
1701
+	unsigned int fill_level = 0;
1702
+
1703
+	efx_for_each_channel_tx_queue(tx_queue, channel)
1704
+		fill_level = max(fill_level,
1705
+				 tx_queue->insert_count - tx_queue->old_read_count);
1706
+
1707
+	return fill_level;
1708
+}
1709
+
15891710 /* Get all supported features.
15901711  * If a feature is not fixed, it is present in hw_features.
15911712  * If a feature is fixed, it does not present in hw_features, but