~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,33 +1,6 @@
	1	+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
1	2	/*
2		- * Copyright 2015 Amazon.com, Inc. or its affiliates.
3		- *
4		- * This software is available to you under a choice of one of two
5		- * licenses. You may choose to be licensed under the terms of the GNU
6		- * General Public License (GPL) Version 2, available from the file
7		- * COPYING in the main directory of this source tree, or the
8		- * BSD license below:
9		- *
10		- * Redistribution and use in source and binary forms, with or
11		- * without modification, are permitted provided that the following
12		- * conditions are met:
13		- *
14		- * - Redistributions of source code must retain the above
15		- * copyright notice, this list of conditions and the following
16		- * disclaimer.
17		- *
18		- * - Redistributions in binary form must reproduce the above
19		- * copyright notice, this list of conditions and the following
20		- * disclaimer in the documentation and/or other materials
21		- * provided with the distribution.
22		- *
23		- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24		- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25		- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26		- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27		- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28		- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29		- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30		- * SOFTWARE.
	3	+ * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
31	4	*/
32	5
33	6	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
..	..	@@ -36,10 +9,8 @@
36	9	#include <linux/cpu_rmap.h>
37	10	#endif /* CONFIG_RFS_ACCEL */
38	11	#include <linux/ethtool.h>
39		-#include <linux/if_vlan.h>
40	12	#include <linux/kernel.h>
41	13	#include <linux/module.h>
42		-#include <linux/moduleparam.h>
43	14	#include <linux/numa.h>
44	15	#include <linux/pci.h>
45	16	#include <linux/utsname.h>
..	..	@@ -48,14 +19,12 @@
48	19	#include <net/ip.h>
49	20
50	21	#include "ena_netdev.h"
	22	+#include <linux/bpf_trace.h>
51	23	#include "ena_pci_id_tbl.h"
52		-
53		-static char version[] = DEVICE_NAME " v" DRV_MODULE_VERSION "\n";
54	24
55	25	MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
56	26	MODULE_DESCRIPTION(DEVICE_NAME);
57	27	MODULE_LICENSE("GPL");
58		-MODULE_VERSION(DRV_MODULE_VERSION);
59	28
60	29	/* Time in jiffies before concluding the transmitter is hung. */
61	30	#define TX_TIMEOUT (5 * HZ)
..	..	@@ -79,7 +48,37 @@
79	48	static void ena_destroy_device(struct ena_adapter *adapter, bool graceful);
80	49	static int ena_restore_device(struct ena_adapter *adapter);
81	50
82		-static void ena_tx_timeout(struct net_device *dev)
	51	+static void ena_init_io_rings(struct ena_adapter *adapter,
	52	+ int first_index, int count);
	53	+static void ena_init_napi_in_range(struct ena_adapter *adapter, int first_index,
	54	+ int count);
	55	+static void ena_del_napi_in_range(struct ena_adapter *adapter, int first_index,
	56	+ int count);
	57	+static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid);
	58	+static int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,
	59	+ int first_index,
	60	+ int count);
	61	+static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid);
	62	+static void ena_free_tx_resources(struct ena_adapter *adapter, int qid);
	63	+static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget);
	64	+static void ena_destroy_all_tx_queues(struct ena_adapter *adapter);
	65	+static void ena_free_all_io_tx_resources(struct ena_adapter *adapter);
	66	+static void ena_napi_disable_in_range(struct ena_adapter *adapter,
	67	+ int first_index, int count);
	68	+static void ena_napi_enable_in_range(struct ena_adapter *adapter,
	69	+ int first_index, int count);
	70	+static int ena_up(struct ena_adapter *adapter);
	71	+static void ena_down(struct ena_adapter *adapter);
	72	+static void ena_unmask_interrupt(struct ena_ring *tx_ring,
	73	+ struct ena_ring *rx_ring);
	74	+static void ena_update_ring_numa_node(struct ena_ring *tx_ring,
	75	+ struct ena_ring *rx_ring);
	76	+static void ena_unmap_tx_buff(struct ena_ring *tx_ring,
	77	+ struct ena_tx_buffer *tx_info);
	78	+static int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,
	79	+ int first_index, int count);
	80	+
	81	+static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue)
83	82	{
84	83	struct ena_adapter *adapter = netdev_priv(dev);
85	84
..	..	@@ -102,7 +101,7 @@
102	101	{
103	102	int i;
104	103
105		- for (i = 0; i < adapter->num_queues; i++)
	104	+ for (i = 0; i < adapter->num_io_queues; i++)
106	105	adapter->rx_ring[i].mtu = mtu;
107	106	}
108	107
..	..	@@ -113,7 +112,7 @@
113	112
114	113	ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
115	114	if (!ret) {
116		- netif_dbg(adapter, drv, dev, "set MTU to %d\n", new_mtu);
	115	+ netif_dbg(adapter, drv, dev, "Set MTU to %d\n", new_mtu);
117	116	update_rx_ring_mtu(adapter, new_mtu);
118	117	dev->mtu = new_mtu;
119	118	} else {
..	..	@@ -124,16 +123,466 @@
124	123	return ret;
125	124	}
126	125
	126	+static int ena_xmit_common(struct net_device *dev,
	127	+ struct ena_ring *ring,
	128	+ struct ena_tx_buffer *tx_info,
	129	+ struct ena_com_tx_ctx *ena_tx_ctx,
	130	+ u16 next_to_use,
	131	+ u32 bytes)
	132	+{
	133	+ struct ena_adapter *adapter = netdev_priv(dev);
	134	+ int rc, nb_hw_desc;
	135	+
	136	+ if (unlikely(ena_com_is_doorbell_needed(ring->ena_com_io_sq,
	137	+ ena_tx_ctx))) {
	138	+ netif_dbg(adapter, tx_queued, dev,
	139	+ "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
	140	+ ring->qid);
	141	+ ena_com_write_sq_doorbell(ring->ena_com_io_sq);
	142	+ }
	143	+
	144	+ /* prepare the packet's descriptors to dma engine */
	145	+ rc = ena_com_prepare_tx(ring->ena_com_io_sq, ena_tx_ctx,
	146	+ &nb_hw_desc);
	147	+
	148	+ /* In case there isn't enough space in the queue for the packet,
	149	+ * we simply drop it. All other failure reasons of
	150	+ * ena_com_prepare_tx() are fatal and therefore require a device reset.
	151	+ */
	152	+ if (unlikely(rc)) {
	153	+ netif_err(adapter, tx_queued, dev,
	154	+ "Failed to prepare tx bufs\n");
	155	+ u64_stats_update_begin(&ring->syncp);
	156	+ ring->tx_stats.prepare_ctx_err++;
	157	+ u64_stats_update_end(&ring->syncp);
	158	+ if (rc != -ENOMEM) {
	159	+ adapter->reset_reason =
	160	+ ENA_REGS_RESET_DRIVER_INVALID_STATE;
	161	+ set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
	162	+ }
	163	+ return rc;
	164	+ }
	165	+
	166	+ u64_stats_update_begin(&ring->syncp);
	167	+ ring->tx_stats.cnt++;
	168	+ ring->tx_stats.bytes += bytes;
	169	+ u64_stats_update_end(&ring->syncp);
	170	+
	171	+ tx_info->tx_descs = nb_hw_desc;
	172	+ tx_info->last_jiffies = jiffies;
	173	+ tx_info->print_once = 0;
	174	+
	175	+ ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
	176	+ ring->ring_size);
	177	+ return 0;
	178	+}
	179	+
	180	+/* This is the XDP napi callback. XDP queues use a separate napi callback
	181	+ * than Rx/Tx queues.
	182	+ */
	183	+static int ena_xdp_io_poll(struct napi_struct *napi, int budget)
	184	+{
	185	+ struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
	186	+ u32 xdp_work_done, xdp_budget;
	187	+ struct ena_ring *xdp_ring;
	188	+ int napi_comp_call = 0;
	189	+ int ret;
	190	+
	191	+ xdp_ring = ena_napi->xdp_ring;
	192	+ xdp_ring->first_interrupt = ena_napi->first_interrupt;
	193	+
	194	+ xdp_budget = budget;
	195	+
	196	+ if (!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags) \|\|
	197	+ test_bit(ENA_FLAG_TRIGGER_RESET, &xdp_ring->adapter->flags)) {
	198	+ napi_complete_done(napi, 0);
	199	+ return 0;
	200	+ }
	201	+
	202	+ xdp_work_done = ena_clean_xdp_irq(xdp_ring, xdp_budget);
	203	+
	204	+ /* If the device is about to reset or down, avoid unmask
	205	+ * the interrupt and return 0 so NAPI won't reschedule
	206	+ */
	207	+ if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags))) {
	208	+ napi_complete_done(napi, 0);
	209	+ ret = 0;
	210	+ } else if (xdp_budget > xdp_work_done) {
	211	+ napi_comp_call = 1;
	212	+ if (napi_complete_done(napi, xdp_work_done))
	213	+ ena_unmask_interrupt(xdp_ring, NULL);
	214	+ ena_update_ring_numa_node(xdp_ring, NULL);
	215	+ ret = xdp_work_done;
	216	+ } else {
	217	+ ret = xdp_budget;
	218	+ }
	219	+
	220	+ u64_stats_update_begin(&xdp_ring->syncp);
	221	+ xdp_ring->tx_stats.napi_comp += napi_comp_call;
	222	+ xdp_ring->tx_stats.tx_poll++;
	223	+ u64_stats_update_end(&xdp_ring->syncp);
	224	+
	225	+ return ret;
	226	+}
	227	+
	228	+static int ena_xdp_tx_map_buff(struct ena_ring *xdp_ring,
	229	+ struct ena_tx_buffer *tx_info,
	230	+ struct xdp_buff *xdp,
	231	+ void **push_hdr,
	232	+ u32 *push_len)
	233	+{
	234	+ struct ena_adapter *adapter = xdp_ring->adapter;
	235	+ struct ena_com_buf *ena_buf;
	236	+ dma_addr_t dma = 0;
	237	+ u32 size;
	238	+
	239	+ tx_info->xdpf = xdp_convert_buff_to_frame(xdp);
	240	+ size = tx_info->xdpf->len;
	241	+ ena_buf = tx_info->bufs;
	242	+
	243	+ /* llq push buffer */
	244	+ *push_len = min_t(u32, size, xdp_ring->tx_max_header_size);
	245	+ *push_hdr = tx_info->xdpf->data;
	246	+
	247	+ if (size - *push_len > 0) {
	248	+ dma = dma_map_single(xdp_ring->dev,
	249	+ push_hdr + push_len,
	250	+ size - *push_len,
	251	+ DMA_TO_DEVICE);
	252	+ if (unlikely(dma_mapping_error(xdp_ring->dev, dma)))
	253	+ goto error_report_dma_error;
	254	+
	255	+ tx_info->map_linear_data = 1;
	256	+ tx_info->num_of_bufs = 1;
	257	+ }
	258	+
	259	+ ena_buf->paddr = dma;
	260	+ ena_buf->len = size;
	261	+
	262	+ return 0;
	263	+
	264	+error_report_dma_error:
	265	+ u64_stats_update_begin(&xdp_ring->syncp);
	266	+ xdp_ring->tx_stats.dma_mapping_err++;
	267	+ u64_stats_update_end(&xdp_ring->syncp);
	268	+ netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n");
	269	+
	270	+ xdp_return_frame_rx_napi(tx_info->xdpf);
	271	+ tx_info->xdpf = NULL;
	272	+ tx_info->num_of_bufs = 0;
	273	+
	274	+ return -EINVAL;
	275	+}
	276	+
	277	+static int ena_xdp_xmit_buff(struct net_device *dev,
	278	+ struct xdp_buff *xdp,
	279	+ int qid,
	280	+ struct ena_rx_buffer *rx_info)
	281	+{
	282	+ struct ena_adapter *adapter = netdev_priv(dev);
	283	+ struct ena_com_tx_ctx ena_tx_ctx = {};
	284	+ struct ena_tx_buffer *tx_info;
	285	+ struct ena_ring *xdp_ring;
	286	+ u16 next_to_use, req_id;
	287	+ int rc;
	288	+ void *push_hdr;
	289	+ u32 push_len;
	290	+
	291	+ xdp_ring = &adapter->tx_ring[qid];
	292	+ next_to_use = xdp_ring->next_to_use;
	293	+ req_id = xdp_ring->free_ids[next_to_use];
	294	+ tx_info = &xdp_ring->tx_buffer_info[req_id];
	295	+ tx_info->num_of_bufs = 0;
	296	+ page_ref_inc(rx_info->page);
	297	+ tx_info->xdp_rx_page = rx_info->page;
	298	+
	299	+ rc = ena_xdp_tx_map_buff(xdp_ring, tx_info, xdp, &push_hdr, &push_len);
	300	+ if (unlikely(rc))
	301	+ goto error_drop_packet;
	302	+
	303	+ ena_tx_ctx.ena_bufs = tx_info->bufs;
	304	+ ena_tx_ctx.push_header = push_hdr;
	305	+ ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
	306	+ ena_tx_ctx.req_id = req_id;
	307	+ ena_tx_ctx.header_len = push_len;
	308	+
	309	+ rc = ena_xmit_common(dev,
	310	+ xdp_ring,
	311	+ tx_info,
	312	+ &ena_tx_ctx,
	313	+ next_to_use,
	314	+ xdp->data_end - xdp->data);
	315	+ if (rc)
	316	+ goto error_unmap_dma;
	317	+ /* trigger the dma engine. ena_com_write_sq_doorbell()
	318	+ * has a mb
	319	+ */
	320	+ ena_com_write_sq_doorbell(xdp_ring->ena_com_io_sq);
	321	+ u64_stats_update_begin(&xdp_ring->syncp);
	322	+ xdp_ring->tx_stats.doorbells++;
	323	+ u64_stats_update_end(&xdp_ring->syncp);
	324	+
	325	+ return NETDEV_TX_OK;
	326	+
	327	+error_unmap_dma:
	328	+ ena_unmap_tx_buff(xdp_ring, tx_info);
	329	+ tx_info->xdpf = NULL;
	330	+error_drop_packet:
	331	+ __free_page(tx_info->xdp_rx_page);
	332	+ return NETDEV_TX_OK;
	333	+}
	334	+
	335	+static int ena_xdp_execute(struct ena_ring *rx_ring,
	336	+ struct xdp_buff *xdp,
	337	+ struct ena_rx_buffer *rx_info)
	338	+{
	339	+ struct bpf_prog *xdp_prog;
	340	+ u32 verdict = XDP_PASS;
	341	+ u64 *xdp_stat;
	342	+
	343	+ rcu_read_lock();
	344	+ xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog);
	345	+
	346	+ if (!xdp_prog)
	347	+ goto out;
	348	+
	349	+ verdict = bpf_prog_run_xdp(xdp_prog, xdp);
	350	+
	351	+ if (verdict == XDP_TX) {
	352	+ ena_xdp_xmit_buff(rx_ring->netdev,
	353	+ xdp,
	354	+ rx_ring->qid + rx_ring->adapter->num_io_queues,
	355	+ rx_info);
	356	+
	357	+ xdp_stat = &rx_ring->rx_stats.xdp_tx;
	358	+ } else if (unlikely(verdict == XDP_ABORTED)) {
	359	+ trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
	360	+ xdp_stat = &rx_ring->rx_stats.xdp_aborted;
	361	+ } else if (unlikely(verdict == XDP_DROP)) {
	362	+ xdp_stat = &rx_ring->rx_stats.xdp_drop;
	363	+ } else if (unlikely(verdict == XDP_PASS)) {
	364	+ xdp_stat = &rx_ring->rx_stats.xdp_pass;
	365	+ } else {
	366	+ bpf_warn_invalid_xdp_action(verdict);
	367	+ xdp_stat = &rx_ring->rx_stats.xdp_invalid;
	368	+ }
	369	+
	370	+ u64_stats_update_begin(&rx_ring->syncp);
	371	+ (*xdp_stat)++;
	372	+ u64_stats_update_end(&rx_ring->syncp);
	373	+out:
	374	+ rcu_read_unlock();
	375	+
	376	+ return verdict;
	377	+}
	378	+
	379	+static void ena_init_all_xdp_queues(struct ena_adapter *adapter)
	380	+{
	381	+ adapter->xdp_first_ring = adapter->num_io_queues;
	382	+ adapter->xdp_num_queues = adapter->num_io_queues;
	383	+
	384	+ ena_init_io_rings(adapter,
	385	+ adapter->xdp_first_ring,
	386	+ adapter->xdp_num_queues);
	387	+}
	388	+
	389	+static int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter)
	390	+{
	391	+ int rc = 0;
	392	+
	393	+ rc = ena_setup_tx_resources_in_range(adapter, adapter->xdp_first_ring,
	394	+ adapter->xdp_num_queues);
	395	+ if (rc)
	396	+ goto setup_err;
	397	+
	398	+ rc = ena_create_io_tx_queues_in_range(adapter,
	399	+ adapter->xdp_first_ring,
	400	+ adapter->xdp_num_queues);
	401	+ if (rc)
	402	+ goto create_err;
	403	+
	404	+ return 0;
	405	+
	406	+create_err:
	407	+ ena_free_all_io_tx_resources(adapter);
	408	+setup_err:
	409	+ return rc;
	410	+}
	411	+
	412	+/* Provides a way for both kernel and bpf-prog to know
	413	+ * more about the RX-queue a given XDP frame arrived on.
	414	+ */
	415	+static int ena_xdp_register_rxq_info(struct ena_ring *rx_ring)
	416	+{
	417	+ int rc;
	418	+
	419	+ rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid);
	420	+
	421	+ if (rc) {
	422	+ netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
	423	+ "Failed to register xdp rx queue info. RX queue num %d rc: %d\n",
	424	+ rx_ring->qid, rc);
	425	+ goto err;
	426	+ }
	427	+
	428	+ rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED,
	429	+ NULL);
	430	+
	431	+ if (rc) {
	432	+ netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
	433	+ "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n",
	434	+ rx_ring->qid, rc);
	435	+ xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
	436	+ }
	437	+
	438	+err:
	439	+ return rc;
	440	+}
	441	+
	442	+static void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring)
	443	+{
	444	+ xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq);
	445	+ xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
	446	+}
	447	+
	448	+static void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter,
	449	+ struct bpf_prog *prog,
	450	+ int first, int count)
	451	+{
	452	+ struct ena_ring *rx_ring;
	453	+ int i = 0;
	454	+
	455	+ for (i = first; i < count; i++) {
	456	+ rx_ring = &adapter->rx_ring[i];
	457	+ xchg(&rx_ring->xdp_bpf_prog, prog);
	458	+ if (prog) {
	459	+ ena_xdp_register_rxq_info(rx_ring);
	460	+ rx_ring->rx_headroom = XDP_PACKET_HEADROOM;
	461	+ } else {
	462	+ ena_xdp_unregister_rxq_info(rx_ring);
	463	+ rx_ring->rx_headroom = 0;
	464	+ }
	465	+ }
	466	+}
	467	+
	468	+static void ena_xdp_exchange_program(struct ena_adapter *adapter,
	469	+ struct bpf_prog *prog)
	470	+{
	471	+ struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog);
	472	+
	473	+ ena_xdp_exchange_program_rx_in_range(adapter,
	474	+ prog,
	475	+ 0,
	476	+ adapter->num_io_queues);
	477	+
	478	+ if (old_bpf_prog)
	479	+ bpf_prog_put(old_bpf_prog);
	480	+}
	481	+
	482	+static int ena_destroy_and_free_all_xdp_queues(struct ena_adapter *adapter)
	483	+{
	484	+ bool was_up;
	485	+ int rc;
	486	+
	487	+ was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
	488	+
	489	+ if (was_up)
	490	+ ena_down(adapter);
	491	+
	492	+ adapter->xdp_first_ring = 0;
	493	+ adapter->xdp_num_queues = 0;
	494	+ ena_xdp_exchange_program(adapter, NULL);
	495	+ if (was_up) {
	496	+ rc = ena_up(adapter);
	497	+ if (rc)
	498	+ return rc;
	499	+ }
	500	+ return 0;
	501	+}
	502	+
	503	+static int ena_xdp_set(struct net_device netdev, struct netdev_bpf bpf)
	504	+{
	505	+ struct ena_adapter *adapter = netdev_priv(netdev);
	506	+ struct bpf_prog *prog = bpf->prog;
	507	+ struct bpf_prog *old_bpf_prog;
	508	+ int rc, prev_mtu;
	509	+ bool is_up;
	510	+
	511	+ is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
	512	+ rc = ena_xdp_allowed(adapter);
	513	+ if (rc == ENA_XDP_ALLOWED) {
	514	+ old_bpf_prog = adapter->xdp_bpf_prog;
	515	+ if (prog) {
	516	+ if (!is_up) {
	517	+ ena_init_all_xdp_queues(adapter);
	518	+ } else if (!old_bpf_prog) {
	519	+ ena_down(adapter);
	520	+ ena_init_all_xdp_queues(adapter);
	521	+ }
	522	+ ena_xdp_exchange_program(adapter, prog);
	523	+
	524	+ if (is_up && !old_bpf_prog) {
	525	+ rc = ena_up(adapter);
	526	+ if (rc)
	527	+ return rc;
	528	+ }
	529	+ } else if (old_bpf_prog) {
	530	+ rc = ena_destroy_and_free_all_xdp_queues(adapter);
	531	+ if (rc)
	532	+ return rc;
	533	+ }
	534	+
	535	+ prev_mtu = netdev->max_mtu;
	536	+ netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu;
	537	+
	538	+ if (!old_bpf_prog)
	539	+ netif_info(adapter, drv, adapter->netdev,
	540	+ "XDP program is set, changing the max_mtu from %d to %d",
	541	+ prev_mtu, netdev->max_mtu);
	542	+
	543	+ } else if (rc == ENA_XDP_CURRENT_MTU_TOO_LARGE) {
	544	+ netif_err(adapter, drv, adapter->netdev,
	545	+ "Failed to set xdp program, the current MTU (%d) is larger than the maximum allowed MTU (%lu) while xdp is on",
	546	+ netdev->mtu, ENA_XDP_MAX_MTU);
	547	+ NL_SET_ERR_MSG_MOD(bpf->extack,
	548	+ "Failed to set xdp program, the current MTU is larger than the maximum allowed MTU. Check the dmesg for more info");
	549	+ return -EINVAL;
	550	+ } else if (rc == ENA_XDP_NO_ENOUGH_QUEUES) {
	551	+ netif_err(adapter, drv, adapter->netdev,
	552	+ "Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed channel count. The current queue count (%d), the maximal queue count (%d)\n",
	553	+ adapter->num_io_queues, adapter->max_num_io_queues);
	554	+ NL_SET_ERR_MSG_MOD(bpf->extack,
	555	+ "Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg for more info");
	556	+ return -EINVAL;
	557	+ }
	558	+
	559	+ return 0;
	560	+}
	561	+
	562	+/* This is the main xdp callback, it's used by the kernel to set/unset the xdp
	563	+ * program as well as to query the current xdp program id.
	564	+ */
	565	+static int ena_xdp(struct net_device netdev, struct netdev_bpf bpf)
	566	+{
	567	+ switch (bpf->command) {
	568	+ case XDP_SETUP_PROG:
	569	+ return ena_xdp_set(netdev, bpf);
	570	+ default:
	571	+ return -EINVAL;
	572	+ }
	573	+ return 0;
	574	+}
	575	+
127	576	static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter)
128	577	{
129	578	#ifdef CONFIG_RFS_ACCEL
130	579	u32 i;
131	580	int rc;
132	581
133		- adapter->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(adapter->num_queues);
	582	+ adapter->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(adapter->num_io_queues);
134	583	if (!adapter->netdev->rx_cpu_rmap)
135	584	return -ENOMEM;
136		- for (i = 0; i < adapter->num_queues; i++) {
	585	+ for (i = 0; i < adapter->num_io_queues; i++) {
137	586	int irq_idx = ENA_IO_IRQ_IDX(i);
138	587
139	588	rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap,
..	..	@@ -159,14 +608,14 @@
159	608	ring->adapter = adapter;
160	609	ring->ena_dev = adapter->ena_dev;
161	610	ring->per_napi_packets = 0;
162		- ring->per_napi_bytes = 0;
163	611	ring->cpu = 0;
164	612	ring->first_interrupt = false;
165	613	ring->no_interrupt_event_cnt = 0;
166	614	u64_stats_init(&ring->syncp);
167	615	}
168	616
169		-static void ena_init_io_rings(struct ena_adapter *adapter)
	617	+static void ena_init_io_rings(struct ena_adapter *adapter,
	618	+ int first_index, int count)
170	619	{
171	620	struct ena_com_dev *ena_dev;
172	621	struct ena_ring txr, rxr;
..	..	@@ -174,29 +623,36 @@
174	623
175	624	ena_dev = adapter->ena_dev;
176	625
177		- for (i = 0; i < adapter->num_queues; i++) {
	626	+ for (i = first_index; i < first_index + count; i++) {
178	627	txr = &adapter->tx_ring[i];
179	628	rxr = &adapter->rx_ring[i];
180	629
181		- /* TX/RX common ring state */
	630	+ /* TX common ring state */
182	631	ena_init_io_rings_common(adapter, txr, i);
183		- ena_init_io_rings_common(adapter, rxr, i);
184	632
185	633	/* TX specific ring state */
186		- txr->ring_size = adapter->tx_ring_size;
	634	+ txr->ring_size = adapter->requested_tx_ring_size;
187	635	txr->tx_max_header_size = ena_dev->tx_max_header_size;
188	636	txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
189	637	txr->sgl_size = adapter->max_tx_sgl_size;
190	638	txr->smoothed_interval =
191	639	ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
	640	+ txr->disable_meta_caching = adapter->disable_meta_caching;
192	641
193		- /* RX specific ring state */
194		- rxr->ring_size = adapter->rx_ring_size;
195		- rxr->rx_copybreak = adapter->rx_copybreak;
196		- rxr->sgl_size = adapter->max_rx_sgl_size;
197		- rxr->smoothed_interval =
198		- ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
199		- rxr->empty_rx_queue = 0;
	642	+ /* Don't init RX queues for xdp queues */
	643	+ if (!ENA_IS_XDP_INDEX(adapter, i)) {
	644	+ /* RX common ring state */
	645	+ ena_init_io_rings_common(adapter, rxr, i);
	646	+
	647	+ /* RX specific ring state */
	648	+ rxr->ring_size = adapter->requested_rx_ring_size;
	649	+ rxr->rx_copybreak = adapter->rx_copybreak;
	650	+ rxr->sgl_size = adapter->max_rx_sgl_size;
	651	+ rxr->smoothed_interval =
	652	+ ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
	653	+ rxr->empty_rx_queue = 0;
	654	+ adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
	655	+ }
200	656	}
201	657	}
202	658
..	..	@@ -225,22 +681,28 @@
225	681	if (!tx_ring->tx_buffer_info) {
226	682	tx_ring->tx_buffer_info = vzalloc(size);
227	683	if (!tx_ring->tx_buffer_info)
228		- return -ENOMEM;
	684	+ goto err_tx_buffer_info;
229	685	}
230	686
231	687	size = sizeof(u16) * tx_ring->ring_size;
232		- tx_ring->free_tx_ids = vzalloc_node(size, node);
233		- if (!tx_ring->free_tx_ids) {
234		- tx_ring->free_tx_ids = vzalloc(size);
235		- if (!tx_ring->free_tx_ids) {
236		- vfree(tx_ring->tx_buffer_info);
237		- return -ENOMEM;
238		- }
	688	+ tx_ring->free_ids = vzalloc_node(size, node);
	689	+ if (!tx_ring->free_ids) {
	690	+ tx_ring->free_ids = vzalloc(size);
	691	+ if (!tx_ring->free_ids)
	692	+ goto err_tx_free_ids;
	693	+ }
	694	+
	695	+ size = tx_ring->tx_max_header_size;
	696	+ tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);
	697	+ if (!tx_ring->push_buf_intermediate_buf) {
	698	+ tx_ring->push_buf_intermediate_buf = vzalloc(size);
	699	+ if (!tx_ring->push_buf_intermediate_buf)
	700	+ goto err_push_buf_intermediate_buf;
239	701	}
240	702
241	703	/* Req id ring for TX out of order completions */
242	704	for (i = 0; i < tx_ring->ring_size; i++)
243		- tx_ring->free_tx_ids[i] = i;
	705	+ tx_ring->free_ids[i] = i;
244	706
245	707	/* Reset tx statistics */
246	708	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));
..	..	@@ -249,6 +711,15 @@
249	711	tx_ring->next_to_clean = 0;
250	712	tx_ring->cpu = ena_irq->cpu;
251	713	return 0;
	714	+
	715	+err_push_buf_intermediate_buf:
	716	+ vfree(tx_ring->free_ids);
	717	+ tx_ring->free_ids = NULL;
	718	+err_tx_free_ids:
	719	+ vfree(tx_ring->tx_buffer_info);
	720	+ tx_ring->tx_buffer_info = NULL;
	721	+err_tx_buffer_info:
	722	+ return -ENOMEM;
252	723	}
253	724
254	725	/* ena_free_tx_resources - Free I/O Tx Resources per Queue
..	..	@@ -264,20 +735,20 @@
264	735	vfree(tx_ring->tx_buffer_info);
265	736	tx_ring->tx_buffer_info = NULL;
266	737
267		- vfree(tx_ring->free_tx_ids);
268		- tx_ring->free_tx_ids = NULL;
	738	+ vfree(tx_ring->free_ids);
	739	+ tx_ring->free_ids = NULL;
	740	+
	741	+ vfree(tx_ring->push_buf_intermediate_buf);
	742	+ tx_ring->push_buf_intermediate_buf = NULL;
269	743	}
270	744
271		-/* ena_setup_all_tx_resources - allocate I/O Tx queues resources for All queues
272		- * @adapter: private structure
273		- *
274		- * Return 0 on success, negative on failure
275		- */
276		-static int ena_setup_all_tx_resources(struct ena_adapter *adapter)
	745	+static int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,
	746	+ int first_index,
	747	+ int count)
277	748	{
278	749	int i, rc = 0;
279	750
280		- for (i = 0; i < adapter->num_queues; i++) {
	751	+ for (i = first_index; i < first_index + count; i++) {
281	752	rc = ena_setup_tx_resources(adapter, i);
282	753	if (rc)
283	754	goto err_setup_tx;
..	..	@@ -291,9 +762,18 @@
291	762	"Tx queue %d: allocation failed\n", i);
292	763
293	764	/* rewind the index freeing the rings as we go */
294		- while (i--)
	765	+ while (first_index < i--)
295	766	ena_free_tx_resources(adapter, i);
296	767	return rc;
	768	+}
	769	+
	770	+static void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter,
	771	+ int first_index, int count)
	772	+{
	773	+ int i;
	774	+
	775	+ for (i = first_index; i < first_index + count; i++)
	776	+ ena_free_tx_resources(adapter, i);
297	777	}
298	778
299	779	/* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues
..	..	@@ -303,28 +783,10 @@
303	783	*/
304	784	static void ena_free_all_io_tx_resources(struct ena_adapter *adapter)
305	785	{
306		- int i;
307		-
308		- for (i = 0; i < adapter->num_queues; i++)
309		- ena_free_tx_resources(adapter, i);
310		-}
311		-
312		-static inline int validate_rx_req_id(struct ena_ring *rx_ring, u16 req_id)
313		-{
314		- if (likely(req_id < rx_ring->ring_size))
315		- return 0;
316		-
317		- netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
318		- "Invalid rx req_id: %hu\n", req_id);
319		-
320		- u64_stats_update_begin(&rx_ring->syncp);
321		- rx_ring->rx_stats.bad_req_id++;
322		- u64_stats_update_end(&rx_ring->syncp);
323		-
324		- /* Trigger device reset */
325		- rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
326		- set_bit(ENA_FLAG_TRIGGER_RESET, &rx_ring->adapter->flags);
327		- return -EFAULT;
	786	+ ena_free_all_io_tx_resources_in_range(adapter,
	787	+ 0,
	788	+ adapter->xdp_num_queues +
	789	+ adapter->num_io_queues);
328	790	}
329	791
330	792	/* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)
..	..	@@ -360,18 +822,19 @@
360	822	}
361	823
362	824	size = sizeof(u16) * rx_ring->ring_size;
363		- rx_ring->free_rx_ids = vzalloc_node(size, node);
364		- if (!rx_ring->free_rx_ids) {
365		- rx_ring->free_rx_ids = vzalloc(size);
366		- if (!rx_ring->free_rx_ids) {
	825	+ rx_ring->free_ids = vzalloc_node(size, node);
	826	+ if (!rx_ring->free_ids) {
	827	+ rx_ring->free_ids = vzalloc(size);
	828	+ if (!rx_ring->free_ids) {
367	829	vfree(rx_ring->rx_buffer_info);
	830	+ rx_ring->rx_buffer_info = NULL;
368	831	return -ENOMEM;
369	832	}
370	833	}
371	834
372	835	/* Req id ring for receiving RX pkts out of order */
373	836	for (i = 0; i < rx_ring->ring_size; i++)
374		- rx_ring->free_rx_ids[i] = i;
	837	+ rx_ring->free_ids[i] = i;
375	838
376	839	/* Reset rx statistics */
377	840	memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));
..	..	@@ -397,8 +860,8 @@
397	860	vfree(rx_ring->rx_buffer_info);
398	861	rx_ring->rx_buffer_info = NULL;
399	862
400		- vfree(rx_ring->free_rx_ids);
401		- rx_ring->free_rx_ids = NULL;
	863	+ vfree(rx_ring->free_ids);
	864	+ rx_ring->free_ids = NULL;
402	865	}
403	866
404	867	/* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
..	..	@@ -410,7 +873,7 @@
410	873	{
411	874	int i, rc = 0;
412	875
413		- for (i = 0; i < adapter->num_queues; i++) {
	876	+ for (i = 0; i < adapter->num_io_queues; i++) {
414	877	rc = ena_setup_rx_resources(adapter, i);
415	878	if (rc)
416	879	goto err_setup_rx;
..	..	@@ -438,16 +901,20 @@
438	901	{
439	902	int i;
440	903
441		- for (i = 0; i < adapter->num_queues; i++)
	904	+ for (i = 0; i < adapter->num_io_queues; i++)
442	905	ena_free_rx_resources(adapter, i);
443	906	}
444	907
445		-static inline int ena_alloc_rx_page(struct ena_ring *rx_ring,
	908	+static int ena_alloc_rx_page(struct ena_ring *rx_ring,
446	909	struct ena_rx_buffer *rx_info, gfp_t gfp)
447	910	{
	911	+ int headroom = rx_ring->rx_headroom;
448	912	struct ena_com_buf *ena_buf;
449	913	struct page *page;
450	914	dma_addr_t dma;
	915	+
	916	+ /* restore page offset value in case it has been changed by device */
	917	+ rx_info->page_offset = headroom;
451	918
452	919	/* if previous allocated page is not used */
453	920	if (unlikely(rx_info->page))
..	..	@@ -461,8 +928,11 @@
461	928	return -ENOMEM;
462	929	}
463	930
	931	+ /* To enable NIC-side port-mirroring, AKA SPAN port,
	932	+ * we make the buffer readable from the nic as well
	933	+ */
464	934	dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE,
465		- DMA_FROM_DEVICE);
	935	+ DMA_BIDIRECTIONAL);
466	936	if (unlikely(dma_mapping_error(rx_ring->dev, dma))) {
467	937	u64_stats_update_begin(&rx_ring->syncp);
468	938	rx_ring->rx_stats.dma_mapping_err++;
..	..	@@ -472,13 +942,12 @@
472	942	return -EIO;
473	943	}
474	944	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
475		- "alloc page %p, rx_info %p\n", page, rx_info);
	945	+ "Allocate page %p, rx_info %p\n", page, rx_info);
476	946
477	947	rx_info->page = page;
478		- rx_info->page_offset = 0;
479	948	ena_buf = &rx_info->ena_buf;
480		- ena_buf->paddr = dma;
481		- ena_buf->len = ENA_PAGE_SIZE;
	949	+ ena_buf->paddr = dma + headroom;
	950	+ ena_buf->len = ENA_PAGE_SIZE - headroom;
482	951
483	952	return 0;
484	953	}
..	..	@@ -495,8 +964,9 @@
495	964	return;
496	965	}
497	966
498		- dma_unmap_page(rx_ring->dev, ena_buf->paddr, ENA_PAGE_SIZE,
499		- DMA_FROM_DEVICE);
	967	+ dma_unmap_page(rx_ring->dev, ena_buf->paddr - rx_ring->rx_headroom,
	968	+ ENA_PAGE_SIZE,
	969	+ DMA_BIDIRECTIONAL);
500	970
501	971	__free_page(page);
502	972	rx_info->page = NULL;
..	..	@@ -513,19 +983,15 @@
513	983	for (i = 0; i < num; i++) {
514	984	struct ena_rx_buffer *rx_info;
515	985
516		- req_id = rx_ring->free_rx_ids[next_to_use];
517		- rc = validate_rx_req_id(rx_ring, req_id);
518		- if (unlikely(rc < 0))
519		- break;
	986	+ req_id = rx_ring->free_ids[next_to_use];
520	987
521	988	rx_info = &rx_ring->rx_buffer_info[req_id];
522		-
523	989
524	990	rc = ena_alloc_rx_page(rx_ring, rx_info,
525	991	GFP_ATOMIC \| __GFP_COMP);
526	992	if (unlikely(rc < 0)) {
527	993	netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
528		- "failed to alloc buffer for rx queue %d\n",
	994	+ "Failed to allocate buffer for rx queue %d\n",
529	995	rx_ring->qid);
530	996	break;
531	997	}
..	..	@@ -534,7 +1000,7 @@
534	1000	req_id);
535	1001	if (unlikely(rc)) {
536	1002	netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
537		- "failed to add buffer for rx queue %d\n",
	1003	+ "Failed to add buffer for rx queue %d\n",
538	1004	rx_ring->qid);
539	1005	break;
540	1006	}
..	..	@@ -546,9 +1012,9 @@
546	1012	u64_stats_update_begin(&rx_ring->syncp);
547	1013	rx_ring->rx_stats.refil_partial++;
548	1014	u64_stats_update_end(&rx_ring->syncp);
549		- netdev_warn(rx_ring->netdev,
550		- "refilled rx qid %d with only %d buffers (from %d)\n",
551		- rx_ring->qid, i, num);
	1015	+ netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
	1016	+ "Refilled rx qid %d with only %d buffers (from %d)\n",
	1017	+ rx_ring->qid, i, num);
552	1018	}
553	1019
554	1020	/* ena_com_write_sq_doorbell issues a wmb() */
..	..	@@ -576,21 +1042,20 @@
576	1042
577	1043	/* ena_refill_all_rx_bufs - allocate all queues Rx buffers
578	1044	* @adapter: board private structure
579		- *
580	1045	*/
581	1046	static void ena_refill_all_rx_bufs(struct ena_adapter *adapter)
582	1047	{
583	1048	struct ena_ring *rx_ring;
584	1049	int i, rc, bufs_num;
585	1050
586		- for (i = 0; i < adapter->num_queues; i++) {
	1051	+ for (i = 0; i < adapter->num_io_queues; i++) {
587	1052	rx_ring = &adapter->rx_ring[i];
588	1053	bufs_num = rx_ring->ring_size - 1;
589	1054	rc = ena_refill_rx_bufs(rx_ring, bufs_num);
590	1055
591	1056	if (unlikely(rc != bufs_num))
592	1057	netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
593		- "refilling Queue %d failed. allocated %d buffers from: %d\n",
	1058	+ "Refilling Queue %d failed. allocated %d buffers from: %d\n",
594	1059	i, rc, bufs_num);
595	1060	}
596	1061	}
..	..	@@ -599,8 +1064,38 @@
599	1064	{
600	1065	int i;
601	1066
602		- for (i = 0; i < adapter->num_queues; i++)
	1067	+ for (i = 0; i < adapter->num_io_queues; i++)
603	1068	ena_free_rx_bufs(adapter, i);
	1069	+}
	1070	+
	1071	+static void ena_unmap_tx_buff(struct ena_ring *tx_ring,
	1072	+ struct ena_tx_buffer *tx_info)
	1073	+{
	1074	+ struct ena_com_buf *ena_buf;
	1075	+ u32 cnt;
	1076	+ int i;
	1077	+
	1078	+ ena_buf = tx_info->bufs;
	1079	+ cnt = tx_info->num_of_bufs;
	1080	+
	1081	+ if (unlikely(!cnt))
	1082	+ return;
	1083	+
	1084	+ if (tx_info->map_linear_data) {
	1085	+ dma_unmap_single(tx_ring->dev,
	1086	+ dma_unmap_addr(ena_buf, paddr),
	1087	+ dma_unmap_len(ena_buf, len),
	1088	+ DMA_TO_DEVICE);
	1089	+ ena_buf++;
	1090	+ cnt--;
	1091	+ }
	1092	+
	1093	+ /* unmap remaining mapped pages */
	1094	+ for (i = 0; i < cnt; i++) {
	1095	+ dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
	1096	+ dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
	1097	+ ena_buf++;
	1098	+ }
604	1099	}
605	1100
606	1101	/* ena_free_tx_bufs - Free Tx Buffers per Queue
..	..	@@ -613,39 +1108,22 @@
613	1108
614	1109	for (i = 0; i < tx_ring->ring_size; i++) {
615	1110	struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
616		- struct ena_com_buf *ena_buf;
617		- int nr_frags;
618		- int j;
619	1111
620	1112	if (!tx_info->skb)
621	1113	continue;
622	1114
623	1115	if (print_once) {
624		- netdev_notice(tx_ring->netdev,
625		- "free uncompleted tx skb qid %d idx 0x%x\n",
626		- tx_ring->qid, i);
	1116	+ netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev,
	1117	+ "Free uncompleted tx skb qid %d idx 0x%x\n",
	1118	+ tx_ring->qid, i);
627	1119	print_once = false;
628	1120	} else {
629		- netdev_dbg(tx_ring->netdev,
630		- "free uncompleted tx skb qid %d idx 0x%x\n",
631		- tx_ring->qid, i);
	1121	+ netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev,
	1122	+ "Free uncompleted tx skb qid %d idx 0x%x\n",
	1123	+ tx_ring->qid, i);
632	1124	}
633	1125
634		- ena_buf = tx_info->bufs;
635		- dma_unmap_single(tx_ring->dev,
636		- ena_buf->paddr,
637		- ena_buf->len,
638		- DMA_TO_DEVICE);
639		-
640		- /* unmap remaining mapped pages */
641		- nr_frags = tx_info->num_of_bufs - 1;
642		- for (j = 0; j < nr_frags; j++) {
643		- ena_buf++;
644		- dma_unmap_page(tx_ring->dev,
645		- ena_buf->paddr,
646		- ena_buf->len,
647		- DMA_TO_DEVICE);
648		- }
	1126	+ ena_unmap_tx_buff(tx_ring, tx_info);
649	1127
650	1128	dev_kfree_skb_any(tx_info->skb);
651	1129	}
..	..	@@ -658,7 +1136,7 @@
658	1136	struct ena_ring *tx_ring;
659	1137	int i;
660	1138
661		- for (i = 0; i < adapter->num_queues; i++) {
	1139	+ for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
662	1140	tx_ring = &adapter->tx_ring[i];
663	1141	ena_free_tx_bufs(tx_ring);
664	1142	}
..	..	@@ -669,7 +1147,7 @@
669	1147	u16 ena_qid;
670	1148	int i;
671	1149
672		- for (i = 0; i < adapter->num_queues; i++) {
	1150	+ for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
673	1151	ena_qid = ENA_IO_TXQ_IDX(i);
674	1152	ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
675	1153	}
..	..	@@ -680,8 +1158,9 @@
680	1158	u16 ena_qid;
681	1159	int i;
682	1160
683		- for (i = 0; i < adapter->num_queues; i++) {
	1161	+ for (i = 0; i < adapter->num_io_queues; i++) {
684	1162	ena_qid = ENA_IO_RXQ_IDX(i);
	1163	+ cancel_work_sync(&adapter->ena_napi[i].dim.work);
685	1164	ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
686	1165	}
687	1166	}
..	..	@@ -692,31 +1171,52 @@
692	1171	ena_destroy_all_rx_queues(adapter);
693	1172	}
694	1173
695		-static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
	1174	+static int handle_invalid_req_id(struct ena_ring *ring, u16 req_id,
	1175	+ struct ena_tx_buffer *tx_info, bool is_xdp)
696	1176	{
697		- struct ena_tx_buffer *tx_info = NULL;
698		-
699		- if (likely(req_id < tx_ring->ring_size)) {
700		- tx_info = &tx_ring->tx_buffer_info[req_id];
701		- if (likely(tx_info->skb))
702		- return 0;
703		- }
704		-
705	1177	if (tx_info)
706		- netif_err(tx_ring->adapter, tx_done, tx_ring->netdev,
707		- "tx_info doesn't have valid skb\n");
	1178	+ netif_err(ring->adapter,
	1179	+ tx_done,
	1180	+ ring->netdev,
	1181	+ "tx_info doesn't have valid %s",
	1182	+ is_xdp ? "xdp frame" : "skb");
708	1183	else
709		- netif_err(tx_ring->adapter, tx_done, tx_ring->netdev,
710		- "Invalid req_id: %hu\n", req_id);
	1184	+ netif_err(ring->adapter,
	1185	+ tx_done,
	1186	+ ring->netdev,
	1187	+ "Invalid req_id: %hu\n",
	1188	+ req_id);
711	1189
712		- u64_stats_update_begin(&tx_ring->syncp);
713		- tx_ring->tx_stats.bad_req_id++;
714		- u64_stats_update_end(&tx_ring->syncp);
	1190	+ u64_stats_update_begin(&ring->syncp);
	1191	+ ring->tx_stats.bad_req_id++;
	1192	+ u64_stats_update_end(&ring->syncp);
715	1193
716	1194	/* Trigger device reset */
717		- tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
718		- set_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags);
	1195	+ ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
	1196	+ set_bit(ENA_FLAG_TRIGGER_RESET, &ring->adapter->flags);
719	1197	return -EFAULT;
	1198	+}
	1199	+
	1200	+static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
	1201	+{
	1202	+ struct ena_tx_buffer *tx_info;
	1203	+
	1204	+ tx_info = &tx_ring->tx_buffer_info[req_id];
	1205	+ if (likely(tx_info->skb))
	1206	+ return 0;
	1207	+
	1208	+ return handle_invalid_req_id(tx_ring, req_id, tx_info, false);
	1209	+}
	1210	+
	1211	+static int validate_xdp_req_id(struct ena_ring *xdp_ring, u16 req_id)
	1212	+{
	1213	+ struct ena_tx_buffer *tx_info;
	1214	+
	1215	+ tx_info = &xdp_ring->tx_buffer_info[req_id];
	1216	+ if (likely(tx_info->xdpf))
	1217	+ return 0;
	1218	+
	1219	+ return handle_invalid_req_id(xdp_ring, req_id, tx_info, true);
720	1220	}
721	1221
722	1222	static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
..	..	@@ -736,14 +1236,17 @@
736	1236	while (tx_pkts < budget) {
737	1237	struct ena_tx_buffer *tx_info;
738	1238	struct sk_buff *skb;
739		- struct ena_com_buf *ena_buf;
740		- int i, nr_frags;
741	1239
742	1240	rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
743	1241	&req_id);
744		- if (rc)
	1242	+ if (rc) {
	1243	+ if (unlikely(rc == -EINVAL))
	1244	+ handle_invalid_req_id(tx_ring, req_id, NULL,
	1245	+ false);
745	1246	break;
	1247	+ }
746	1248
	1249	+ /* validate that the request id points to a valid skb */
747	1250	rc = validate_tx_req_id(tx_ring, req_id);
748	1251	if (rc)
749	1252	break;
..	..	@@ -757,24 +1260,7 @@
757	1260	tx_info->skb = NULL;
758	1261	tx_info->last_jiffies = 0;
759	1262
760		- if (likely(tx_info->num_of_bufs != 0)) {
761		- ena_buf = tx_info->bufs;
762		-
763		- dma_unmap_single(tx_ring->dev,
764		- dma_unmap_addr(ena_buf, paddr),
765		- dma_unmap_len(ena_buf, len),
766		- DMA_TO_DEVICE);
767		-
768		- /* unmap remaining mapped pages */
769		- nr_frags = tx_info->num_of_bufs - 1;
770		- for (i = 0; i < nr_frags; i++) {
771		- ena_buf++;
772		- dma_unmap_page(tx_ring->dev,
773		- dma_unmap_addr(ena_buf, paddr),
774		- dma_unmap_len(ena_buf, len),
775		- DMA_TO_DEVICE);
776		- }
777		- }
	1263	+ ena_unmap_tx_buff(tx_ring, tx_info);
778	1264
779	1265	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
780	1266	"tx_poll: q %d skb %p completed\n", tx_ring->qid,
..	..	@@ -785,7 +1271,7 @@
785	1271	tx_pkts++;
786	1272	total_done += tx_info->tx_descs;
787	1273
788		- tx_ring->free_tx_ids[next_to_clean] = req_id;
	1274	+ tx_ring->free_ids[next_to_clean] = req_id;
789	1275	next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
790	1276	tx_ring->ring_size);
791	1277	}
..	..	@@ -805,13 +1291,15 @@
805	1291	*/
806	1292	smp_mb();
807	1293
808		- above_thresh = ena_com_sq_empty_space(tx_ring->ena_com_io_sq) >
809		- ENA_TX_WAKEUP_THRESH;
	1294	+ above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
	1295	+ ENA_TX_WAKEUP_THRESH);
810	1296	if (unlikely(netif_tx_queue_stopped(txq) && above_thresh)) {
811	1297	__netif_tx_lock(txq, smp_processor_id());
812		- above_thresh = ena_com_sq_empty_space(tx_ring->ena_com_io_sq) >
813		- ENA_TX_WAKEUP_THRESH;
814		- if (netif_tx_queue_stopped(txq) && above_thresh) {
	1298	+ above_thresh =
	1299	+ ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
	1300	+ ENA_TX_WAKEUP_THRESH);
	1301	+ if (netif_tx_queue_stopped(txq) && above_thresh &&
	1302	+ test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {
815	1303	netif_tx_wake_queue(txq);
816	1304	u64_stats_update_begin(&tx_ring->syncp);
817	1305	tx_ring->tx_stats.queue_wakeup++;
..	..	@@ -819,9 +1307,6 @@
819	1307	}
820	1308	__netif_tx_unlock(txq);
821	1309	}
822		-
823		- tx_ring->per_napi_bytes += tx_bytes;
824		- tx_ring->per_napi_packets += tx_pkts;
825	1310
826	1311	return tx_pkts;
827	1312	}
..	..	@@ -860,6 +1345,7 @@
860	1345
861	1346	len = ena_bufs[buf].len;
862	1347	req_id = ena_bufs[buf].req_id;
	1348	+
863	1349	rx_info = &rx_ring->rx_buffer_info[req_id];
864	1350
865	1351	if (unlikely(!rx_info->page)) {
..	..	@@ -874,7 +1360,8 @@
874	1360
875	1361	/* save virt address of first buffer */
876	1362	va = page_address(rx_info->page) + rx_info->page_offset;
877		- prefetch(va + NET_IP_ALIGN);
	1363	+
	1364	+ prefetch(va);
878	1365
879	1366	if (len <= rx_ring->rx_copybreak) {
880	1367	skb = ena_alloc_skb(rx_ring, false);
..	..	@@ -882,7 +1369,7 @@
882	1369	return NULL;
883	1370
884	1371	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
885		- "rx allocated small packet. len %d. data_len %d\n",
	1372	+ "RX allocated small packet. len %d. data_len %d\n",
886	1373	skb->len, skb->data_len);
887	1374
888	1375	/* sync this buffer for CPU use */
..	..	@@ -898,7 +1385,7 @@
898	1385
899	1386	skb_put(skb, len);
900	1387	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
901		- rx_ring->free_rx_ids[*next_to_clean] = req_id;
	1388	+ rx_ring->free_ids[*next_to_clean] = req_id;
902	1389	next_to_clean = ENA_RX_RING_IDX_ADD(next_to_clean, descs,
903	1390	rx_ring->ring_size);
904	1391	return skb;
..	..	@@ -911,18 +1398,18 @@
911	1398	do {
912	1399	dma_unmap_page(rx_ring->dev,
913	1400	dma_unmap_addr(&rx_info->ena_buf, paddr),
914		- ENA_PAGE_SIZE, DMA_FROM_DEVICE);
	1401	+ ENA_PAGE_SIZE, DMA_BIDIRECTIONAL);
915	1402
916	1403	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page,
917	1404	rx_info->page_offset, len, ENA_PAGE_SIZE);
918	1405
919	1406	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
920		- "rx skb updated. len %d. data_len %d\n",
	1407	+ "RX skb updated. len %d. data_len %d\n",
921	1408	skb->len, skb->data_len);
922	1409
923	1410	rx_info->page = NULL;
924	1411
925		- rx_ring->free_rx_ids[*next_to_clean] = req_id;
	1412	+ rx_ring->free_ids[*next_to_clean] = req_id;
926	1413	*next_to_clean =
927	1414	ENA_RX_RING_IDX_NEXT(*next_to_clean,
928	1415	rx_ring->ring_size);
..	..	@@ -932,6 +1419,7 @@
932	1419	buf++;
933	1420	len = ena_bufs[buf].len;
934	1421	req_id = ena_bufs[buf].req_id;
	1422	+
935	1423	rx_info = &rx_ring->rx_buffer_info[req_id];
936	1424	} while (1);
937	1425
..	..	@@ -943,7 +1431,7 @@
943	1431	* @ena_rx_ctx: received packet context/metadata
944	1432	* @skb: skb currently being received and modified
945	1433	*/
946		-static inline void ena_rx_checksum(struct ena_ring *rx_ring,
	1434	+static void ena_rx_checksum(struct ena_ring *rx_ring,
947	1435	struct ena_com_rx_ctx *ena_rx_ctx,
948	1436	struct sk_buff *skb)
949	1437	{
..	..	@@ -986,8 +1474,22 @@
986	1474	return;
987	1475	}
988	1476
989		- skb->ip_summed = CHECKSUM_UNNECESSARY;
	1477	+ if (likely(ena_rx_ctx->l4_csum_checked)) {
	1478	+ skb->ip_summed = CHECKSUM_UNNECESSARY;
	1479	+ u64_stats_update_begin(&rx_ring->syncp);
	1480	+ rx_ring->rx_stats.csum_good++;
	1481	+ u64_stats_update_end(&rx_ring->syncp);
	1482	+ } else {
	1483	+ u64_stats_update_begin(&rx_ring->syncp);
	1484	+ rx_ring->rx_stats.csum_unchecked++;
	1485	+ u64_stats_update_end(&rx_ring->syncp);
	1486	+ skb->ip_summed = CHECKSUM_NONE;
	1487	+ }
	1488	+ } else {
	1489	+ skb->ip_summed = CHECKSUM_NONE;
	1490	+ return;
990	1491	}
	1492	+
991	1493	}
992	1494
993	1495	static void ena_set_rx_hash(struct ena_ring *rx_ring,
..	..	@@ -1012,6 +1514,32 @@
1012	1514	}
1013	1515	}
1014	1516
	1517	+static int ena_xdp_handle_buff(struct ena_ring rx_ring, struct xdp_buff xdp)
	1518	+{
	1519	+ struct ena_rx_buffer *rx_info;
	1520	+ int ret;
	1521	+
	1522	+ rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
	1523	+ xdp->data = page_address(rx_info->page) + rx_info->page_offset;
	1524	+ xdp_set_data_meta_invalid(xdp);
	1525	+ xdp->data_hard_start = page_address(rx_info->page);
	1526	+ xdp->data_end = xdp->data + rx_ring->ena_bufs[0].len;
	1527	+ /* If for some reason we received a bigger packet than
	1528	+ * we expect, then we simply drop it
	1529	+ */
	1530	+ if (unlikely(rx_ring->ena_bufs[0].len > ENA_XDP_MAX_MTU))
	1531	+ return XDP_DROP;
	1532	+
	1533	+ ret = ena_xdp_execute(rx_ring, xdp, rx_info);
	1534	+
	1535	+ /* The xdp program might expand the headers */
	1536	+ if (ret == XDP_PASS) {
	1537	+ rx_info->page_offset = xdp->data - xdp->data_hard_start;
	1538	+ rx_ring->ena_bufs[0].len = xdp->data_end - xdp->data;
	1539	+ }
	1540	+
	1541	+ return ret;
	1542	+}
1015	1543	/* ena_clean_rx_irq - Cleanup RX irq
1016	1544	* @rx_ring: RX ring to clean
1017	1545	* @napi: napi handler
..	..	@@ -1023,26 +1551,33 @@
1023	1551	u32 budget)
1024	1552	{
1025	1553	u16 next_to_clean = rx_ring->next_to_clean;
1026		- u32 res_budget, work_done;
1027		-
1028	1554	struct ena_com_rx_ctx ena_rx_ctx;
	1555	+ struct ena_rx_buffer *rx_info;
1029	1556	struct ena_adapter *adapter;
	1557	+ u32 res_budget, work_done;
	1558	+ int rx_copybreak_pkt = 0;
	1559	+ int refill_threshold;
1030	1560	struct sk_buff *skb;
1031	1561	int refill_required;
1032		- int refill_threshold;
1033		- int rc = 0;
	1562	+ struct xdp_buff xdp;
1034	1563	int total_len = 0;
1035		- int rx_copybreak_pkt = 0;
	1564	+ int xdp_verdict;
	1565	+ int rc = 0;
1036	1566	int i;
1037	1567
1038	1568	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1039	1569	"%s qid %d\n", __func__, rx_ring->qid);
1040	1570	res_budget = budget;
	1571	+ xdp.rxq = &rx_ring->xdp_rxq;
	1572	+ xdp.frame_sz = ENA_PAGE_SIZE;
1041	1573
1042	1574	do {
	1575	+ xdp_verdict = XDP_PASS;
	1576	+ skb = NULL;
1043	1577	ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
1044	1578	ena_rx_ctx.max_bufs = rx_ring->sgl_size;
1045	1579	ena_rx_ctx.descs = 0;
	1580	+ ena_rx_ctx.pkt_offset = 0;
1046	1581	rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,
1047	1582	rx_ring->ena_com_io_sq,
1048	1583	&ena_rx_ctx);
..	..	@@ -1052,23 +1587,44 @@
1052	1587	if (unlikely(ena_rx_ctx.descs == 0))
1053	1588	break;
1054	1589
	1590	+ /* First descriptor might have an offset set by the device */
	1591	+ rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
	1592	+ rx_info->page_offset += ena_rx_ctx.pkt_offset;
	1593	+
1055	1594	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1056	1595	"rx_poll: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",
1057	1596	rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
1058	1597	ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
1059	1598
1060		- /* allocate skb and fill it */
1061		- skb = ena_rx_skb(rx_ring, rx_ring->ena_bufs, ena_rx_ctx.descs,
1062		- &next_to_clean);
	1599	+ if (ena_xdp_present_ring(rx_ring))
	1600	+ xdp_verdict = ena_xdp_handle_buff(rx_ring, &xdp);
1063	1601
1064		- /* exit if we failed to retrieve a buffer */
	1602	+ /* allocate skb and fill it */
	1603	+ if (xdp_verdict == XDP_PASS)
	1604	+ skb = ena_rx_skb(rx_ring,
	1605	+ rx_ring->ena_bufs,
	1606	+ ena_rx_ctx.descs,
	1607	+ &next_to_clean);
	1608	+
1065	1609	if (unlikely(!skb)) {
	1610	+ /* The page might not actually be freed here since the
	1611	+ * page reference count is incremented in
	1612	+ * ena_xdp_xmit_buff(), and it will be decreased only
	1613	+ * when send completion was received from the device
	1614	+ */
	1615	+ if (xdp_verdict == XDP_TX)
	1616	+ ena_free_rx_page(rx_ring,
	1617	+ &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id]);
1066	1618	for (i = 0; i < ena_rx_ctx.descs; i++) {
1067		- rx_ring->free_tx_ids[next_to_clean] =
	1619	+ rx_ring->free_ids[next_to_clean] =
1068	1620	rx_ring->ena_bufs[i].req_id;
1069	1621	next_to_clean =
1070	1622	ENA_RX_RING_IDX_NEXT(next_to_clean,
1071	1623	rx_ring->ring_size);
	1624	+ }
	1625	+ if (xdp_verdict != XDP_PASS) {
	1626	+ res_budget--;
	1627	+ continue;
1072	1628	}
1073	1629	break;
1074	1630	}
..	..	@@ -1092,7 +1648,6 @@
1092	1648	} while (likely(res_budget));
1093	1649
1094	1650	work_done = budget - res_budget;
1095		- rx_ring->per_napi_bytes += total_len;
1096	1651	rx_ring->per_napi_packets += work_done;
1097	1652	u64_stats_update_begin(&rx_ring->syncp);
1098	1653	rx_ring->rx_stats.bytes += total_len;
..	..	@@ -1102,8 +1657,10 @@
1102	1657
1103	1658	rx_ring->next_to_clean = next_to_clean;
1104	1659
1105		- refill_required = ena_com_sq_empty_space(rx_ring->ena_com_io_sq);
1106		- refill_threshold = rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER;
	1660	+ refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
	1661	+ refill_threshold =
	1662	+ min_t(int, rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
	1663	+ ENA_RX_REFILL_THRESH_PACKET);
1107	1664
1108	1665	/* Optimization, try to batch new rx buffers */
1109	1666	if (refill_required > refill_threshold) {
..	..	@@ -1116,57 +1673,88 @@
1116	1673	error:
1117	1674	adapter = netdev_priv(rx_ring->netdev);
1118	1675
1119		- u64_stats_update_begin(&rx_ring->syncp);
1120		- rx_ring->rx_stats.bad_desc_num++;
1121		- u64_stats_update_end(&rx_ring->syncp);
	1676	+ if (rc == -ENOSPC) {
	1677	+ u64_stats_update_begin(&rx_ring->syncp);
	1678	+ rx_ring->rx_stats.bad_desc_num++;
	1679	+ u64_stats_update_end(&rx_ring->syncp);
	1680	+ adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
	1681	+ } else {
	1682	+ u64_stats_update_begin(&rx_ring->syncp);
	1683	+ rx_ring->rx_stats.bad_req_id++;
	1684	+ u64_stats_update_end(&rx_ring->syncp);
	1685	+ adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
	1686	+ }
1122	1687
1123		- /* Too many desc from the device. Trigger reset */
1124		- adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
1125	1688	set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
1126	1689
1127	1690	return 0;
1128	1691	}
1129	1692
1130		-inline void ena_adjust_intr_moderation(struct ena_ring *rx_ring,
1131		- struct ena_ring *tx_ring)
	1693	+static void ena_dim_work(struct work_struct *w)
1132	1694	{
1133		- /* We apply adaptive moderation on Rx path only.
1134		- * Tx uses static interrupt moderation.
1135		- */
1136		- ena_com_calculate_interrupt_delay(rx_ring->ena_dev,
1137		- rx_ring->per_napi_packets,
1138		- rx_ring->per_napi_bytes,
1139		- &rx_ring->smoothed_interval,
1140		- &rx_ring->moder_tbl_idx);
	1695	+ struct dim *dim = container_of(w, struct dim, work);
	1696	+ struct dim_cq_moder cur_moder =
	1697	+ net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
	1698	+ struct ena_napi *ena_napi = container_of(dim, struct ena_napi, dim);
1141	1699
1142		- /* Reset per napi packets/bytes */
1143		- tx_ring->per_napi_packets = 0;
1144		- tx_ring->per_napi_bytes = 0;
1145		- rx_ring->per_napi_packets = 0;
1146		- rx_ring->per_napi_bytes = 0;
	1700	+ ena_napi->rx_ring->smoothed_interval = cur_moder.usec;
	1701	+ dim->state = DIM_START_MEASURE;
1147	1702	}
1148	1703
1149		-static inline void ena_unmask_interrupt(struct ena_ring *tx_ring,
	1704	+static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi)
	1705	+{
	1706	+ struct dim_sample dim_sample;
	1707	+ struct ena_ring *rx_ring = ena_napi->rx_ring;
	1708	+
	1709	+ if (!rx_ring->per_napi_packets)
	1710	+ return;
	1711	+
	1712	+ rx_ring->non_empty_napi_events++;
	1713	+
	1714	+ dim_update_sample(rx_ring->non_empty_napi_events,
	1715	+ rx_ring->rx_stats.cnt,
	1716	+ rx_ring->rx_stats.bytes,
	1717	+ &dim_sample);
	1718	+
	1719	+ net_dim(&ena_napi->dim, dim_sample);
	1720	+
	1721	+ rx_ring->per_napi_packets = 0;
	1722	+}
	1723	+
	1724	+static void ena_unmask_interrupt(struct ena_ring *tx_ring,
1150	1725	struct ena_ring *rx_ring)
1151	1726	{
1152	1727	struct ena_eth_io_intr_reg intr_reg;
	1728	+ u32 rx_interval = 0;
	1729	+ /* Rx ring can be NULL when for XDP tx queues which don't have an
	1730	+ * accompanying rx_ring pair.
	1731	+ */
	1732	+ if (rx_ring)
	1733	+ rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ?
	1734	+ rx_ring->smoothed_interval :
	1735	+ ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev);
1153	1736
1154	1737	/* Update intr register: rx intr delay,
1155	1738	* tx intr delay and interrupt unmask
1156	1739	*/
1157	1740	ena_com_update_intr_reg(&intr_reg,
1158		- rx_ring->smoothed_interval,
	1741	+ rx_interval,
1159	1742	tx_ring->smoothed_interval,
1160	1743	true);
	1744	+
	1745	+ u64_stats_update_begin(&tx_ring->syncp);
	1746	+ tx_ring->tx_stats.unmask_interrupt++;
	1747	+ u64_stats_update_end(&tx_ring->syncp);
1161	1748
1162	1749	/* It is a shared MSI-X.
1163	1750	* Tx and Rx CQ have pointer to it.
1164	1751	* So we use one of them to reach the intr reg
	1752	+ * The Tx ring is used because the rx_ring is NULL for XDP queues
1165	1753	*/
1166		- ena_com_unmask_intr(rx_ring->ena_com_io_cq, &intr_reg);
	1754	+ ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg);
1167	1755	}
1168	1756
1169		-static inline void ena_update_ring_numa_node(struct ena_ring *tx_ring,
	1757	+static void ena_update_ring_numa_node(struct ena_ring *tx_ring,
1170	1758	struct ena_ring *rx_ring)
1171	1759	{
1172	1760	int cpu = get_cpu();
..	..	@@ -1181,22 +1769,87 @@
1181	1769
1182	1770	if (numa_node != NUMA_NO_NODE) {
1183	1771	ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);
1184		- ena_com_update_numa_node(rx_ring->ena_com_io_cq, numa_node);
	1772	+ if (rx_ring)
	1773	+ ena_com_update_numa_node(rx_ring->ena_com_io_cq,
	1774	+ numa_node);
1185	1775	}
1186	1776
1187	1777	tx_ring->cpu = cpu;
1188		- rx_ring->cpu = cpu;
	1778	+ if (rx_ring)
	1779	+ rx_ring->cpu = cpu;
1189	1780
1190	1781	return;
1191	1782	out:
1192	1783	put_cpu();
1193	1784	}
1194	1785
	1786	+static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget)
	1787	+{
	1788	+ u32 total_done = 0;
	1789	+ u16 next_to_clean;
	1790	+ u32 tx_bytes = 0;
	1791	+ int tx_pkts = 0;
	1792	+ u16 req_id;
	1793	+ int rc;
	1794	+
	1795	+ if (unlikely(!xdp_ring))
	1796	+ return 0;
	1797	+ next_to_clean = xdp_ring->next_to_clean;
	1798	+
	1799	+ while (tx_pkts < budget) {
	1800	+ struct ena_tx_buffer *tx_info;
	1801	+ struct xdp_frame *xdpf;
	1802	+
	1803	+ rc = ena_com_tx_comp_req_id_get(xdp_ring->ena_com_io_cq,
	1804	+ &req_id);
	1805	+ if (rc) {
	1806	+ if (unlikely(rc == -EINVAL))
	1807	+ handle_invalid_req_id(xdp_ring, req_id, NULL,
	1808	+ true);
	1809	+ break;
	1810	+ }
	1811	+
	1812	+ /* validate that the request id points to a valid xdp_frame */
	1813	+ rc = validate_xdp_req_id(xdp_ring, req_id);
	1814	+ if (rc)
	1815	+ break;
	1816	+
	1817	+ tx_info = &xdp_ring->tx_buffer_info[req_id];
	1818	+ xdpf = tx_info->xdpf;
	1819	+
	1820	+ tx_info->xdpf = NULL;
	1821	+ tx_info->last_jiffies = 0;
	1822	+ ena_unmap_tx_buff(xdp_ring, tx_info);
	1823	+
	1824	+ netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev,
	1825	+ "tx_poll: q %d skb %p completed\n", xdp_ring->qid,
	1826	+ xdpf);
	1827	+
	1828	+ tx_bytes += xdpf->len;
	1829	+ tx_pkts++;
	1830	+ total_done += tx_info->tx_descs;
	1831	+
	1832	+ __free_page(tx_info->xdp_rx_page);
	1833	+ xdp_ring->free_ids[next_to_clean] = req_id;
	1834	+ next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
	1835	+ xdp_ring->ring_size);
	1836	+ }
	1837	+
	1838	+ xdp_ring->next_to_clean = next_to_clean;
	1839	+ ena_com_comp_ack(xdp_ring->ena_com_io_sq, total_done);
	1840	+ ena_com_update_dev_comp_head(xdp_ring->ena_com_io_cq);
	1841	+
	1842	+ netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev,
	1843	+ "tx_poll: q %d done. total pkts: %d\n",
	1844	+ xdp_ring->qid, tx_pkts);
	1845	+
	1846	+ return tx_pkts;
	1847	+}
	1848	+
1195	1849	static int ena_io_poll(struct napi_struct *napi, int budget)
1196	1850	{
1197	1851	struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
1198	1852	struct ena_ring tx_ring, rx_ring;
1199		-
1200	1853	int tx_work_done;
1201	1854	int rx_work_done = 0;
1202	1855	int tx_budget;
..	..	@@ -1205,6 +1858,9 @@
1205	1858
1206	1859	tx_ring = ena_napi->tx_ring;
1207	1860	rx_ring = ena_napi->rx_ring;
	1861	+
	1862	+ tx_ring->first_interrupt = ena_napi->first_interrupt;
	1863	+ rx_ring->first_interrupt = ena_napi->first_interrupt;
1208	1864
1209	1865	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;
1210	1866
..	..	@@ -1235,10 +1891,15 @@
1235	1891	/* Update numa and unmask the interrupt only when schedule
1236	1892	* from the interrupt context (vs from sk_busy_loop)
1237	1893	*/
1238		- if (napi_complete_done(napi, rx_work_done)) {
1239		- /* Tx and Rx share the same interrupt vector */
	1894	+ if (napi_complete_done(napi, rx_work_done) &&
	1895	+ READ_ONCE(ena_napi->interrupts_masked)) {
	1896	+ smp_rmb(); /* make sure interrupts_masked is read */
	1897	+ WRITE_ONCE(ena_napi->interrupts_masked, false);
	1898	+ /* We apply adaptive moderation on Rx path only.
	1899	+ * Tx uses static interrupt moderation.
	1900	+ */
1240	1901	if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
1241		- ena_adjust_intr_moderation(rx_ring, tx_ring);
	1902	+ ena_adjust_adaptive_rx_intr_moderation(ena_napi);
1242	1903
1243	1904	ena_unmask_interrupt(tx_ring, rx_ring);
1244	1905	}
..	..	@@ -1279,8 +1940,10 @@
1279	1940	{
1280	1941	struct ena_napi *ena_napi = data;
1281	1942
1282		- ena_napi->tx_ring->first_interrupt = true;
1283		- ena_napi->rx_ring->first_interrupt = true;
	1943	+ ena_napi->first_interrupt = true;
	1944	+
	1945	+ WRITE_ONCE(ena_napi->interrupts_masked, true);
	1946	+ smp_wmb(); /* write interrupts_masked before calling napi */
1284	1947
1285	1948	napi_schedule_irqoff(&ena_napi->napi);
1286	1949
..	..	@@ -1292,7 +1955,7 @@
1292	1955	* the number of potential io queues is the minimum of what the device
1293	1956	* supports and the number of vCPUs.
1294	1957	*/
1295		-static int ena_enable_msix(struct ena_adapter *adapter, int num_queues)
	1958	+static int ena_enable_msix(struct ena_adapter *adapter)
1296	1959	{
1297	1960	int msix_vecs, irq_cnt;
1298	1961
..	..	@@ -1303,10 +1966,9 @@
1303	1966	}
1304	1967
1305	1968	/* Reserved the max msix vectors we might need */
1306		- msix_vecs = ENA_MAX_MSIX_VEC(num_queues);
1307		-
	1969	+ msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues);
1308	1970	netif_dbg(adapter, probe, adapter->netdev,
1309		- "trying to enable MSI-X, vectors %d\n", msix_vecs);
	1971	+ "Trying to enable MSI-X, vectors %d\n", msix_vecs);
1310	1972
1311	1973	irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC,
1312	1974	msix_vecs, PCI_IRQ_MSIX);
..	..	@@ -1319,9 +1981,9 @@
1319	1981
1320	1982	if (irq_cnt != msix_vecs) {
1321	1983	netif_notice(adapter, probe, adapter->netdev,
1322		- "enable only %d MSI-X (out of %d), reduce the number of queues\n",
	1984	+ "Enable only %d MSI-X (out of %d), reduce the number of queues\n",
1323	1985	irq_cnt, msix_vecs);
1324		- adapter->num_queues = irq_cnt - ENA_ADMIN_MSIX_VEC;
	1986	+ adapter->num_io_queues = irq_cnt - ENA_ADMIN_MSIX_VEC;
1325	1987	}
1326	1988
1327	1989	if (ena_init_rx_cpu_rmap(adapter))
..	..	@@ -1356,10 +2018,12 @@
1356	2018	{
1357	2019	struct net_device *netdev;
1358	2020	int irq_idx, i, cpu;
	2021	+ int io_queue_count;
1359	2022
1360	2023	netdev = adapter->netdev;
	2024	+ io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1361	2025
1362		- for (i = 0; i < adapter->num_queues; i++) {
	2026	+ for (i = 0; i < io_queue_count; i++) {
1363	2027	irq_idx = ENA_IO_IRQ_IDX(i);
1364	2028	cpu = i % num_online_cpus();
1365	2029
..	..	@@ -1387,12 +2051,12 @@
1387	2051	irq->data);
1388	2052	if (rc) {
1389	2053	netif_err(adapter, probe, adapter->netdev,
1390		- "failed to request admin irq\n");
	2054	+ "Failed to request admin irq\n");
1391	2055	return rc;
1392	2056	}
1393	2057
1394	2058	netif_dbg(adapter, probe, adapter->netdev,
1395		- "set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n",
	2059	+ "Set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n",
1396	2060	irq->affinity_hint_mask.bits[0], irq->vector);
1397	2061
1398	2062	irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
..	..	@@ -1402,6 +2066,7 @@
1402	2066
1403	2067	static int ena_request_io_irq(struct ena_adapter *adapter)
1404	2068	{
	2069	+ u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1405	2070	unsigned long flags = 0;
1406	2071	struct ena_irq *irq;
1407	2072	int rc = 0, i, k;
..	..	@@ -1412,7 +2077,7 @@
1412	2077	return -EINVAL;
1413	2078	}
1414	2079
1415		- for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
	2080	+ for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
1416	2081	irq = &adapter->irq_tbl[i];
1417	2082	rc = request_irq(irq->vector, irq->handler, flags, irq->name,
1418	2083	irq->data);
..	..	@@ -1424,7 +2089,7 @@
1424	2089	}
1425	2090
1426	2091	netif_dbg(adapter, ifup, adapter->netdev,
1427		- "set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n",
	2092	+ "Set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n",
1428	2093	i, irq->affinity_hint_mask.bits[0], irq->vector);
1429	2094
1430	2095	irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
..	..	@@ -1453,6 +2118,7 @@
1453	2118
1454	2119	static void ena_free_io_irq(struct ena_adapter *adapter)
1455	2120	{
	2121	+ u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1456	2122	struct ena_irq *irq;
1457	2123	int i;
1458	2124
..	..	@@ -1463,7 +2129,7 @@
1463	2129	}
1464	2130	#endif /* CONFIG_RFS_ACCEL */
1465	2131
1466		- for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
	2132	+ for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
1467	2133	irq = &adapter->irq_tbl[i];
1468	2134	irq_set_affinity_hint(irq->vector, NULL);
1469	2135	free_irq(irq->vector, irq->data);
..	..	@@ -1478,63 +2144,71 @@
1478	2144
1479	2145	static void ena_disable_io_intr_sync(struct ena_adapter *adapter)
1480	2146	{
	2147	+ u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1481	2148	int i;
1482	2149
1483	2150	if (!netif_running(adapter->netdev))
1484	2151	return;
1485	2152
1486		- for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++)
	2153	+ for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++)
1487	2154	synchronize_irq(adapter->irq_tbl[i].vector);
1488	2155	}
1489	2156
1490		-static void ena_del_napi(struct ena_adapter *adapter)
	2157	+static void ena_del_napi_in_range(struct ena_adapter *adapter,
	2158	+ int first_index,
	2159	+ int count)
1491	2160	{
1492	2161	int i;
1493	2162
1494		- for (i = 0; i < adapter->num_queues; i++)
	2163	+ for (i = first_index; i < first_index + count; i++) {
1495	2164	netif_napi_del(&adapter->ena_napi[i].napi);
	2165	+
	2166	+ WARN_ON(!ENA_IS_XDP_INDEX(adapter, i) &&
	2167	+ adapter->ena_napi[i].xdp_ring);
	2168	+ }
1496	2169	}
1497	2170
1498		-static void ena_init_napi(struct ena_adapter *adapter)
	2171	+static void ena_init_napi_in_range(struct ena_adapter *adapter,
	2172	+ int first_index, int count)
1499	2173	{
1500		- struct ena_napi *napi;
1501	2174	int i;
1502	2175
1503		- for (i = 0; i < adapter->num_queues; i++) {
1504		- napi = &adapter->ena_napi[i];
	2176	+ for (i = first_index; i < first_index + count; i++) {
	2177	+ struct ena_napi *napi = &adapter->ena_napi[i];
1505	2178
1506	2179	netif_napi_add(adapter->netdev,
1507		- &adapter->ena_napi[i].napi,
1508		- ena_io_poll,
	2180	+ &napi->napi,
	2181	+ ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll,
1509	2182	ENA_NAPI_BUDGET);
1510		- napi->rx_ring = &adapter->rx_ring[i];
1511		- napi->tx_ring = &adapter->tx_ring[i];
	2183	+
	2184	+ if (!ENA_IS_XDP_INDEX(adapter, i)) {
	2185	+ napi->rx_ring = &adapter->rx_ring[i];
	2186	+ napi->tx_ring = &adapter->tx_ring[i];
	2187	+ } else {
	2188	+ napi->xdp_ring = &adapter->tx_ring[i];
	2189	+ }
1512	2190	napi->qid = i;
1513	2191	}
1514	2192	}
1515	2193
1516		-static void ena_napi_disable_all(struct ena_adapter *adapter)
	2194	+static void ena_napi_disable_in_range(struct ena_adapter *adapter,
	2195	+ int first_index,
	2196	+ int count)
1517	2197	{
1518	2198	int i;
1519	2199
1520		- for (i = 0; i < adapter->num_queues; i++)
	2200	+ for (i = first_index; i < first_index + count; i++)
1521	2201	napi_disable(&adapter->ena_napi[i].napi);
1522	2202	}
1523	2203
1524		-static void ena_napi_enable_all(struct ena_adapter *adapter)
	2204	+static void ena_napi_enable_in_range(struct ena_adapter *adapter,
	2205	+ int first_index,
	2206	+ int count)
1525	2207	{
1526	2208	int i;
1527	2209
1528		- for (i = 0; i < adapter->num_queues; i++)
	2210	+ for (i = first_index; i < first_index + count; i++)
1529	2211	napi_enable(&adapter->ena_napi[i].napi);
1530		-}
1531		-
1532		-static void ena_restore_ethtool_params(struct ena_adapter *adapter)
1533		-{
1534		- adapter->tx_usecs = 0;
1535		- adapter->rx_usecs = 0;
1536		- adapter->tx_frames = 1;
1537		- adapter->rx_frames = 1;
1538	2212	}
1539	2213
1540	2214	/* Configure the Rx forwarding */
..	..	@@ -1586,16 +2260,16 @@
1586	2260	/* enable transmits */
1587	2261	netif_tx_start_all_queues(adapter->netdev);
1588	2262
1589		- ena_restore_ethtool_params(adapter);
1590		-
1591		- ena_napi_enable_all(adapter);
	2263	+ ena_napi_enable_in_range(adapter,
	2264	+ 0,
	2265	+ adapter->xdp_num_queues + adapter->num_io_queues);
1592	2266
1593	2267	return 0;
1594	2268	}
1595	2269
1596	2270	static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
1597	2271	{
1598		- struct ena_com_create_io_ctx ctx = { 0 };
	2272	+ struct ena_com_create_io_ctx ctx;
1599	2273	struct ena_com_dev *ena_dev;
1600	2274	struct ena_ring *tx_ring;
1601	2275	u32 msix_vector;
..	..	@@ -1608,11 +2282,13 @@
1608	2282	msix_vector = ENA_IO_IRQ_IDX(qid);
1609	2283	ena_qid = ENA_IO_TXQ_IDX(qid);
1610	2284
	2285	+ memset(&ctx, 0x0, sizeof(ctx));
	2286	+
1611	2287	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1612	2288	ctx.qid = ena_qid;
1613	2289	ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1614	2290	ctx.msix_vector = msix_vector;
1615		- ctx.queue_size = adapter->tx_ring_size;
	2291	+ ctx.queue_size = tx_ring->ring_size;
1616	2292	ctx.numa_node = cpu_to_node(tx_ring->cpu);
1617	2293
1618	2294	rc = ena_com_create_io_queue(ena_dev, &ctx);
..	..	@@ -1638,12 +2314,13 @@
1638	2314	return rc;
1639	2315	}
1640	2316
1641		-static int ena_create_all_io_tx_queues(struct ena_adapter *adapter)
	2317	+static int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,
	2318	+ int first_index, int count)
1642	2319	{
1643	2320	struct ena_com_dev *ena_dev = adapter->ena_dev;
1644	2321	int rc, i;
1645	2322
1646		- for (i = 0; i < adapter->num_queues; i++) {
	2323	+ for (i = first_index; i < first_index + count; i++) {
1647	2324	rc = ena_create_io_tx_queue(adapter, i);
1648	2325	if (rc)
1649	2326	goto create_err;
..	..	@@ -1652,7 +2329,7 @@
1652	2329	return 0;
1653	2330
1654	2331	create_err:
1655		- while (i--)
	2332	+ while (i-- > first_index)
1656	2333	ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
1657	2334
1658	2335	return rc;
..	..	@@ -1661,7 +2338,7 @@
1661	2338	static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
1662	2339	{
1663	2340	struct ena_com_dev *ena_dev;
1664		- struct ena_com_create_io_ctx ctx = { 0 };
	2341	+ struct ena_com_create_io_ctx ctx;
1665	2342	struct ena_ring *rx_ring;
1666	2343	u32 msix_vector;
1667	2344	u16 ena_qid;
..	..	@@ -1673,11 +2350,13 @@
1673	2350	msix_vector = ENA_IO_IRQ_IDX(qid);
1674	2351	ena_qid = ENA_IO_RXQ_IDX(qid);
1675	2352
	2353	+ memset(&ctx, 0x0, sizeof(ctx));
	2354	+
1676	2355	ctx.qid = ena_qid;
1677	2356	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1678	2357	ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1679	2358	ctx.msix_vector = msix_vector;
1680		- ctx.queue_size = adapter->rx_ring_size;
	2359	+ ctx.queue_size = rx_ring->ring_size;
1681	2360	ctx.numa_node = cpu_to_node(rx_ring->cpu);
1682	2361
1683	2362	rc = ena_com_create_io_queue(ena_dev, &ctx);
..	..	@@ -1695,12 +2374,14 @@
1695	2374	netif_err(adapter, ifup, adapter->netdev,
1696	2375	"Failed to get RX queue handlers. RX queue num %d rc: %d\n",
1697	2376	qid, rc);
1698		- ena_com_destroy_io_queue(ena_dev, ena_qid);
1699		- return rc;
	2377	+ goto err;
1700	2378	}
1701	2379
1702	2380	ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);
1703	2381
	2382	+ return rc;
	2383	+err:
	2384	+ ena_com_destroy_io_queue(ena_dev, ena_qid);
1704	2385	return rc;
1705	2386	}
1706	2387
..	..	@@ -1709,27 +2390,148 @@
1709	2390	struct ena_com_dev *ena_dev = adapter->ena_dev;
1710	2391	int rc, i;
1711	2392
1712		- for (i = 0; i < adapter->num_queues; i++) {
	2393	+ for (i = 0; i < adapter->num_io_queues; i++) {
1713	2394	rc = ena_create_io_rx_queue(adapter, i);
1714	2395	if (rc)
1715	2396	goto create_err;
	2397	+ INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work);
1716	2398	}
1717	2399
1718	2400	return 0;
1719	2401
1720	2402	create_err:
1721		- while (i--)
	2403	+ while (i--) {
	2404	+ cancel_work_sync(&adapter->ena_napi[i].dim.work);
1722	2405	ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
	2406	+ }
1723	2407
1724	2408	return rc;
1725	2409	}
1726	2410
	2411	+static void set_io_rings_size(struct ena_adapter *adapter,
	2412	+ int new_tx_size,
	2413	+ int new_rx_size)
	2414	+{
	2415	+ int i;
	2416	+
	2417	+ for (i = 0; i < adapter->num_io_queues; i++) {
	2418	+ adapter->tx_ring[i].ring_size = new_tx_size;
	2419	+ adapter->rx_ring[i].ring_size = new_rx_size;
	2420	+ }
	2421	+}
	2422	+
	2423	+/* This function allows queue allocation to backoff when the system is
	2424	+ * low on memory. If there is not enough memory to allocate io queues
	2425	+ * the driver will try to allocate smaller queues.
	2426	+ *
	2427	+ * The backoff algorithm is as follows:
	2428	+ * 1. Try to allocate TX and RX and if successful.
	2429	+ * 1.1. return success
	2430	+ *
	2431	+ * 2. Divide by 2 the size of the larger of RX and TX queues (or both if their size is the same).
	2432	+ *
	2433	+ * 3. If TX or RX is smaller than 256
	2434	+ * 3.1. return failure.
	2435	+ * 4. else
	2436	+ * 4.1. go back to 1.
	2437	+ */
	2438	+static int create_queues_with_size_backoff(struct ena_adapter *adapter)
	2439	+{
	2440	+ int rc, cur_rx_ring_size, cur_tx_ring_size;
	2441	+ int new_rx_ring_size, new_tx_ring_size;
	2442	+
	2443	+ /* current queue sizes might be set to smaller than the requested
	2444	+ * ones due to past queue allocation failures.
	2445	+ */
	2446	+ set_io_rings_size(adapter, adapter->requested_tx_ring_size,
	2447	+ adapter->requested_rx_ring_size);
	2448	+
	2449	+ while (1) {
	2450	+ if (ena_xdp_present(adapter)) {
	2451	+ rc = ena_setup_and_create_all_xdp_queues(adapter);
	2452	+
	2453	+ if (rc)
	2454	+ goto err_setup_tx;
	2455	+ }
	2456	+ rc = ena_setup_tx_resources_in_range(adapter,
	2457	+ 0,
	2458	+ adapter->num_io_queues);
	2459	+ if (rc)
	2460	+ goto err_setup_tx;
	2461	+
	2462	+ rc = ena_create_io_tx_queues_in_range(adapter,
	2463	+ 0,
	2464	+ adapter->num_io_queues);
	2465	+ if (rc)
	2466	+ goto err_create_tx_queues;
	2467	+
	2468	+ rc = ena_setup_all_rx_resources(adapter);
	2469	+ if (rc)
	2470	+ goto err_setup_rx;
	2471	+
	2472	+ rc = ena_create_all_io_rx_queues(adapter);
	2473	+ if (rc)
	2474	+ goto err_create_rx_queues;
	2475	+
	2476	+ return 0;
	2477	+
	2478	+err_create_rx_queues:
	2479	+ ena_free_all_io_rx_resources(adapter);
	2480	+err_setup_rx:
	2481	+ ena_destroy_all_tx_queues(adapter);
	2482	+err_create_tx_queues:
	2483	+ ena_free_all_io_tx_resources(adapter);
	2484	+err_setup_tx:
	2485	+ if (rc != -ENOMEM) {
	2486	+ netif_err(adapter, ifup, adapter->netdev,
	2487	+ "Queue creation failed with error code %d\n",
	2488	+ rc);
	2489	+ return rc;
	2490	+ }
	2491	+
	2492	+ cur_tx_ring_size = adapter->tx_ring[0].ring_size;
	2493	+ cur_rx_ring_size = adapter->rx_ring[0].ring_size;
	2494	+
	2495	+ netif_err(adapter, ifup, adapter->netdev,
	2496	+ "Not enough memory to create queues with sizes TX=%d, RX=%d\n",
	2497	+ cur_tx_ring_size, cur_rx_ring_size);
	2498	+
	2499	+ new_tx_ring_size = cur_tx_ring_size;
	2500	+ new_rx_ring_size = cur_rx_ring_size;
	2501	+
	2502	+ /* Decrease the size of the larger queue, or
	2503	+ * decrease both if they are the same size.
	2504	+ */
	2505	+ if (cur_rx_ring_size <= cur_tx_ring_size)
	2506	+ new_tx_ring_size = cur_tx_ring_size / 2;
	2507	+ if (cur_rx_ring_size >= cur_tx_ring_size)
	2508	+ new_rx_ring_size = cur_rx_ring_size / 2;
	2509	+
	2510	+ if (new_tx_ring_size < ENA_MIN_RING_SIZE \|\|
	2511	+ new_rx_ring_size < ENA_MIN_RING_SIZE) {
	2512	+ netif_err(adapter, ifup, adapter->netdev,
	2513	+ "Queue creation failed with the smallest possible queue size of %d for both queues. Not retrying with smaller queues\n",
	2514	+ ENA_MIN_RING_SIZE);
	2515	+ return rc;
	2516	+ }
	2517	+
	2518	+ netif_err(adapter, ifup, adapter->netdev,
	2519	+ "Retrying queue creation with sizes TX=%d, RX=%d\n",
	2520	+ new_tx_ring_size,
	2521	+ new_rx_ring_size);
	2522	+
	2523	+ set_io_rings_size(adapter, new_tx_ring_size,
	2524	+ new_rx_ring_size);
	2525	+ }
	2526	+}
	2527	+
1727	2528	static int ena_up(struct ena_adapter *adapter)
1728	2529	{
1729		- int rc, i;
	2530	+ int io_queue_count, rc, i;
1730	2531
1731		- netdev_dbg(adapter->netdev, "%s\n", __func__);
	2532	+ netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);
1732	2533
	2534	+ io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1733	2535	ena_setup_io_intr(adapter);
1734	2536
1735	2537	/* napi poll functions should be initialized before running
..	..	@@ -1737,31 +2539,15 @@
1737	2539	* interrupt, causing the ISR to fire immediately while the poll
1738	2540	* function wasn't set yet, causing a null dereference
1739	2541	*/
1740		- ena_init_napi(adapter);
	2542	+ ena_init_napi_in_range(adapter, 0, io_queue_count);
1741	2543
1742	2544	rc = ena_request_io_irq(adapter);
1743	2545	if (rc)
1744	2546	goto err_req_irq;
1745	2547
1746		- /* allocate transmit descriptors */
1747		- rc = ena_setup_all_tx_resources(adapter);
	2548	+ rc = create_queues_with_size_backoff(adapter);
1748	2549	if (rc)
1749		- goto err_setup_tx;
1750		-
1751		- /* allocate receive descriptors */
1752		- rc = ena_setup_all_rx_resources(adapter);
1753		- if (rc)
1754		- goto err_setup_rx;
1755		-
1756		- /* Create TX queues */
1757		- rc = ena_create_all_io_tx_queues(adapter);
1758		- if (rc)
1759		- goto err_create_tx_queues;
1760		-
1761		- /* Create RX queues */
1762		- rc = ena_create_all_io_rx_queues(adapter);
1763		- if (rc)
1764		- goto err_create_rx_queues;
	2550	+ goto err_create_queues_with_backoff;
1765	2551
1766	2552	rc = ena_up_complete(adapter);
1767	2553	if (rc)
..	..	@@ -1777,36 +2563,35 @@
1777	2563	set_bit(ENA_FLAG_DEV_UP, &adapter->flags);
1778	2564
1779	2565	/* Enable completion queues interrupt */
1780		- for (i = 0; i < adapter->num_queues; i++)
	2566	+ for (i = 0; i < adapter->num_io_queues; i++)
1781	2567	ena_unmask_interrupt(&adapter->tx_ring[i],
1782	2568	&adapter->rx_ring[i]);
1783	2569
1784	2570	/* schedule napi in case we had pending packets
1785	2571	* from the last time we disable napi
1786	2572	*/
1787		- for (i = 0; i < adapter->num_queues; i++)
	2573	+ for (i = 0; i < io_queue_count; i++)
1788	2574	napi_schedule(&adapter->ena_napi[i].napi);
1789	2575
1790	2576	return rc;
1791	2577
1792	2578	err_up:
1793		- ena_destroy_all_rx_queues(adapter);
1794		-err_create_rx_queues:
1795	2579	ena_destroy_all_tx_queues(adapter);
1796		-err_create_tx_queues:
1797		- ena_free_all_io_rx_resources(adapter);
1798		-err_setup_rx:
1799	2580	ena_free_all_io_tx_resources(adapter);
1800		-err_setup_tx:
	2581	+ ena_destroy_all_rx_queues(adapter);
	2582	+ ena_free_all_io_rx_resources(adapter);
	2583	+err_create_queues_with_backoff:
1801	2584	ena_free_io_irq(adapter);
1802	2585	err_req_irq:
1803		- ena_del_napi(adapter);
	2586	+ ena_del_napi_in_range(adapter, 0, io_queue_count);
1804	2587
1805	2588	return rc;
1806	2589	}
1807	2590
1808	2591	static void ena_down(struct ena_adapter *adapter)
1809	2592	{
	2593	+ int io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
	2594	+
1810	2595	netif_info(adapter, ifdown, adapter->netdev, "%s\n", __func__);
1811	2596
1812	2597	clear_bit(ENA_FLAG_DEV_UP, &adapter->flags);
..	..	@@ -1819,7 +2604,7 @@
1819	2604	netif_tx_disable(adapter->netdev);
1820	2605
1821	2606	/* After this point the napi handler won't enable the tx queue */
1822		- ena_napi_disable_all(adapter);
	2607	+ ena_napi_disable_in_range(adapter, 0, io_queue_count);
1823	2608
1824	2609	/* After destroy the queue there won't be any new interrupts */
1825	2610
..	..	@@ -1828,14 +2613,17 @@
1828	2613
1829	2614	rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
1830	2615	if (rc)
1831		- dev_err(&adapter->pdev->dev, "Device reset failed\n");
	2616	+ netif_err(adapter, ifdown, adapter->netdev,
	2617	+ "Device reset failed\n");
	2618	+ /* stop submitting admin commands on a device that was reset */
	2619	+ ena_com_set_admin_running_state(adapter->ena_dev, false);
1832	2620	}
1833	2621
1834	2622	ena_destroy_all_io_queues(adapter);
1835	2623
1836	2624	ena_disable_io_intr_sync(adapter);
1837	2625	ena_free_io_irq(adapter);
1838		- ena_del_napi(adapter);
	2626	+ ena_del_napi_in_range(adapter, 0, io_queue_count);
1839	2627
1840	2628	ena_free_all_tx_bufs(adapter);
1841	2629	ena_free_all_rx_bufs(adapter);
..	..	@@ -1860,13 +2648,13 @@
1860	2648	int rc;
1861	2649
1862	2650	/* Notify the stack of the actual queue counts. */
1863		- rc = netif_set_real_num_tx_queues(netdev, adapter->num_queues);
	2651	+ rc = netif_set_real_num_tx_queues(netdev, adapter->num_io_queues);
1864	2652	if (rc) {
1865	2653	netif_err(adapter, ifup, netdev, "Can't set num tx queues\n");
1866	2654	return rc;
1867	2655	}
1868	2656
1869		- rc = netif_set_real_num_rx_queues(netdev, adapter->num_queues);
	2657	+ rc = netif_set_real_num_rx_queues(netdev, adapter->num_io_queues);
1870	2658	if (rc) {
1871	2659	netif_err(adapter, ifup, netdev, "Can't set num rx queues\n");
1872	2660	return rc;
..	..	@@ -1895,6 +2683,9 @@
1895	2683
1896	2684	netif_dbg(adapter, ifdown, netdev, "%s\n", __func__);
1897	2685
	2686	+ if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
	2687	+ return 0;
	2688	+
1898	2689	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
1899	2690	ena_down(adapter);
1900	2691
..	..	@@ -1912,7 +2703,63 @@
1912	2703	return 0;
1913	2704	}
1914	2705
1915		-static void ena_tx_csum(struct ena_com_tx_ctx ena_tx_ctx, struct sk_buff skb)
	2706	+int ena_update_queue_sizes(struct ena_adapter *adapter,
	2707	+ u32 new_tx_size,
	2708	+ u32 new_rx_size)
	2709	+{
	2710	+ bool dev_was_up;
	2711	+
	2712	+ dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
	2713	+ ena_close(adapter->netdev);
	2714	+ adapter->requested_tx_ring_size = new_tx_size;
	2715	+ adapter->requested_rx_ring_size = new_rx_size;
	2716	+ ena_init_io_rings(adapter,
	2717	+ 0,
	2718	+ adapter->xdp_num_queues +
	2719	+ adapter->num_io_queues);
	2720	+ return dev_was_up ? ena_up(adapter) : 0;
	2721	+}
	2722	+
	2723	+int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count)
	2724	+{
	2725	+ struct ena_com_dev *ena_dev = adapter->ena_dev;
	2726	+ int prev_channel_count;
	2727	+ bool dev_was_up;
	2728	+
	2729	+ dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
	2730	+ ena_close(adapter->netdev);
	2731	+ prev_channel_count = adapter->num_io_queues;
	2732	+ adapter->num_io_queues = new_channel_count;
	2733	+ if (ena_xdp_present(adapter) &&
	2734	+ ena_xdp_allowed(adapter) == ENA_XDP_ALLOWED) {
	2735	+ adapter->xdp_first_ring = new_channel_count;
	2736	+ adapter->xdp_num_queues = new_channel_count;
	2737	+ if (prev_channel_count > new_channel_count)
	2738	+ ena_xdp_exchange_program_rx_in_range(adapter,
	2739	+ NULL,
	2740	+ new_channel_count,
	2741	+ prev_channel_count);
	2742	+ else
	2743	+ ena_xdp_exchange_program_rx_in_range(adapter,
	2744	+ adapter->xdp_bpf_prog,
	2745	+ prev_channel_count,
	2746	+ new_channel_count);
	2747	+ }
	2748	+
	2749	+ /* We need to destroy the rss table so that the indirection
	2750	+ * table will be reinitialized by ena_up()
	2751	+ */
	2752	+ ena_com_rss_destroy(ena_dev);
	2753	+ ena_init_io_rings(adapter,
	2754	+ 0,
	2755	+ adapter->xdp_num_queues +
	2756	+ adapter->num_io_queues);
	2757	+ return dev_was_up ? ena_open(adapter->netdev) : 0;
	2758	+}
	2759	+
	2760	+static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx,
	2761	+ struct sk_buff *skb,
	2762	+ bool disable_meta_caching)
1916	2763	{
1917	2764	u32 mss = skb_shinfo(skb)->gso_size;
1918	2765	struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
..	..	@@ -1956,7 +2803,9 @@
1956	2803	ena_meta->l3_hdr_len = skb_network_header_len(skb);
1957	2804	ena_meta->l3_hdr_offset = skb_network_offset(skb);
1958	2805	ena_tx_ctx->meta_valid = 1;
1959		-
	2806	+ } else if (disable_meta_caching) {
	2807	+ memset(ena_meta, 0, sizeof(*ena_meta));
	2808	+ ena_tx_ctx->meta_valid = 1;
1960	2809	} else {
1961	2810	ena_tx_ctx->meta_valid = 0;
1962	2811	}
..	..	@@ -1991,6 +2840,112 @@
1991	2840	return rc;
1992	2841	}
1993	2842
	2843	+static int ena_tx_map_skb(struct ena_ring *tx_ring,
	2844	+ struct ena_tx_buffer *tx_info,
	2845	+ struct sk_buff *skb,
	2846	+ void **push_hdr,
	2847	+ u16 *header_len)
	2848	+{
	2849	+ struct ena_adapter *adapter = tx_ring->adapter;
	2850	+ struct ena_com_buf *ena_buf;
	2851	+ dma_addr_t dma;
	2852	+ u32 skb_head_len, frag_len, last_frag;
	2853	+ u16 push_len = 0;
	2854	+ u16 delta = 0;
	2855	+ int i = 0;
	2856	+
	2857	+ skb_head_len = skb_headlen(skb);
	2858	+ tx_info->skb = skb;
	2859	+ ena_buf = tx_info->bufs;
	2860	+
	2861	+ if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
	2862	+ /* When the device is LLQ mode, the driver will copy
	2863	+ * the header into the device memory space.
	2864	+ * the ena_com layer assume the header is in a linear
	2865	+ * memory space.
	2866	+ * This assumption might be wrong since part of the header
	2867	+ * can be in the fragmented buffers.
	2868	+ * Use skb_header_pointer to make sure the header is in a
	2869	+ * linear memory space.
	2870	+ */
	2871	+
	2872	+ push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);
	2873	+ *push_hdr = skb_header_pointer(skb, 0, push_len,
	2874	+ tx_ring->push_buf_intermediate_buf);
	2875	+ *header_len = push_len;
	2876	+ if (unlikely(skb->data != *push_hdr)) {
	2877	+ u64_stats_update_begin(&tx_ring->syncp);
	2878	+ tx_ring->tx_stats.llq_buffer_copy++;
	2879	+ u64_stats_update_end(&tx_ring->syncp);
	2880	+
	2881	+ delta = push_len - skb_head_len;
	2882	+ }
	2883	+ } else {
	2884	+ *push_hdr = NULL;
	2885	+ *header_len = min_t(u32, skb_head_len,
	2886	+ tx_ring->tx_max_header_size);
	2887	+ }
	2888	+
	2889	+ netif_dbg(adapter, tx_queued, adapter->netdev,
	2890	+ "skb: %p header_buf->vaddr: %p push_len: %d\n", skb,
	2891	+ *push_hdr, push_len);
	2892	+
	2893	+ if (skb_head_len > push_len) {
	2894	+ dma = dma_map_single(tx_ring->dev, skb->data + push_len,
	2895	+ skb_head_len - push_len, DMA_TO_DEVICE);
	2896	+ if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
	2897	+ goto error_report_dma_error;
	2898	+
	2899	+ ena_buf->paddr = dma;
	2900	+ ena_buf->len = skb_head_len - push_len;
	2901	+
	2902	+ ena_buf++;
	2903	+ tx_info->num_of_bufs++;
	2904	+ tx_info->map_linear_data = 1;
	2905	+ } else {
	2906	+ tx_info->map_linear_data = 0;
	2907	+ }
	2908	+
	2909	+ last_frag = skb_shinfo(skb)->nr_frags;
	2910	+
	2911	+ for (i = 0; i < last_frag; i++) {
	2912	+ const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
	2913	+
	2914	+ frag_len = skb_frag_size(frag);
	2915	+
	2916	+ if (unlikely(delta >= frag_len)) {
	2917	+ delta -= frag_len;
	2918	+ continue;
	2919	+ }
	2920	+
	2921	+ dma = skb_frag_dma_map(tx_ring->dev, frag, delta,
	2922	+ frag_len - delta, DMA_TO_DEVICE);
	2923	+ if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
	2924	+ goto error_report_dma_error;
	2925	+
	2926	+ ena_buf->paddr = dma;
	2927	+ ena_buf->len = frag_len - delta;
	2928	+ ena_buf++;
	2929	+ tx_info->num_of_bufs++;
	2930	+ delta = 0;
	2931	+ }
	2932	+
	2933	+ return 0;
	2934	+
	2935	+error_report_dma_error:
	2936	+ u64_stats_update_begin(&tx_ring->syncp);
	2937	+ tx_ring->tx_stats.dma_mapping_err++;
	2938	+ u64_stats_update_end(&tx_ring->syncp);
	2939	+ netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map skb\n");
	2940	+
	2941	+ tx_info->skb = NULL;
	2942	+
	2943	+ tx_info->num_of_bufs += i;
	2944	+ ena_unmap_tx_buff(tx_ring, tx_info);
	2945	+
	2946	+ return -EINVAL;
	2947	+}
	2948	+
1994	2949	/* Called with netif_tx_lock. */
1995	2950	static netdev_tx_t ena_start_xmit(struct sk_buff skb, struct net_device dev)
1996	2951	{
..	..	@@ -1999,16 +2954,9 @@
1999	2954	struct ena_com_tx_ctx ena_tx_ctx;
2000	2955	struct ena_ring *tx_ring;
2001	2956	struct netdev_queue *txq;
2002		- struct ena_com_buf *ena_buf;
2003	2957	void *push_hdr;
2004		- u32 len, last_frag;
2005		- u16 next_to_use;
2006		- u16 req_id;
2007		- u16 push_len;
2008		- u16 header_len;
2009		- dma_addr_t dma;
2010		- int qid, rc, nb_hw_desc;
2011		- int i = -1;
	2958	+ u16 next_to_use, req_id, header_len;
	2959	+ int qid, rc;
2012	2960
2013	2961	netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);
2014	2962	/* Determine which tx ring we will be placed on */
..	..	@@ -2021,62 +2969,17 @@
2021	2969	goto error_drop_packet;
2022	2970
2023	2971	skb_tx_timestamp(skb);
2024		- len = skb_headlen(skb);
2025	2972
2026	2973	next_to_use = tx_ring->next_to_use;
2027		- req_id = tx_ring->free_tx_ids[next_to_use];
	2974	+ req_id = tx_ring->free_ids[next_to_use];
2028	2975	tx_info = &tx_ring->tx_buffer_info[req_id];
2029	2976	tx_info->num_of_bufs = 0;
2030	2977
2031	2978	WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);
2032		- ena_buf = tx_info->bufs;
2033		- tx_info->skb = skb;
2034	2979
2035		- if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2036		- /* prepared the push buffer */
2037		- push_len = min_t(u32, len, tx_ring->tx_max_header_size);
2038		- header_len = push_len;
2039		- push_hdr = skb->data;
2040		- } else {
2041		- push_len = 0;
2042		- header_len = min_t(u32, len, tx_ring->tx_max_header_size);
2043		- push_hdr = NULL;
2044		- }
2045		-
2046		- netif_dbg(adapter, tx_queued, dev,
2047		- "skb: %p header_buf->vaddr: %p push_len: %d\n", skb,
2048		- push_hdr, push_len);
2049		-
2050		- if (len > push_len) {
2051		- dma = dma_map_single(tx_ring->dev, skb->data + push_len,
2052		- len - push_len, DMA_TO_DEVICE);
2053		- if (dma_mapping_error(tx_ring->dev, dma))
2054		- goto error_report_dma_error;
2055		-
2056		- ena_buf->paddr = dma;
2057		- ena_buf->len = len - push_len;
2058		-
2059		- ena_buf++;
2060		- tx_info->num_of_bufs++;
2061		- }
2062		-
2063		- last_frag = skb_shinfo(skb)->nr_frags;
2064		-
2065		- for (i = 0; i < last_frag; i++) {
2066		- const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2067		-
2068		- len = skb_frag_size(frag);
2069		- dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
2070		- DMA_TO_DEVICE);
2071		- if (dma_mapping_error(tx_ring->dev, dma))
2072		- goto error_report_dma_error;
2073		-
2074		- ena_buf->paddr = dma;
2075		- ena_buf->len = len;
2076		- ena_buf++;
2077		- }
2078		-
2079		- tx_info->num_of_bufs += last_frag;
	2980	+ rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);
	2981	+ if (unlikely(rc))
	2982	+ goto error_drop_packet;
2080	2983
2081	2984	memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
2082	2985	ena_tx_ctx.ena_bufs = tx_info->bufs;
..	..	@@ -2086,43 +2989,25 @@
2086	2989	ena_tx_ctx.header_len = header_len;
2087	2990
2088	2991	/* set flags and meta data */
2089		- ena_tx_csum(&ena_tx_ctx, skb);
	2992	+ ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);
2090	2993
2091		- /* prepare the packet's descriptors to dma engine */
2092		- rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, &ena_tx_ctx,
2093		- &nb_hw_desc);
2094		-
2095		- if (unlikely(rc)) {
2096		- netif_err(adapter, tx_queued, dev,
2097		- "failed to prepare tx bufs\n");
2098		- u64_stats_update_begin(&tx_ring->syncp);
2099		- tx_ring->tx_stats.queue_stop++;
2100		- tx_ring->tx_stats.prepare_ctx_err++;
2101		- u64_stats_update_end(&tx_ring->syncp);
2102		- netif_tx_stop_queue(txq);
	2994	+ rc = ena_xmit_common(dev,
	2995	+ tx_ring,
	2996	+ tx_info,
	2997	+ &ena_tx_ctx,
	2998	+ next_to_use,
	2999	+ skb->len);
	3000	+ if (rc)
2103	3001	goto error_unmap_dma;
2104		- }
2105	3002
2106	3003	netdev_tx_sent_queue(txq, skb->len);
2107		-
2108		- u64_stats_update_begin(&tx_ring->syncp);
2109		- tx_ring->tx_stats.cnt++;
2110		- tx_ring->tx_stats.bytes += skb->len;
2111		- u64_stats_update_end(&tx_ring->syncp);
2112		-
2113		- tx_info->tx_descs = nb_hw_desc;
2114		- tx_info->last_jiffies = jiffies;
2115		- tx_info->print_once = 0;
2116		-
2117		- tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
2118		- tx_ring->ring_size);
2119	3004
2120	3005	/* stop the queue when no more space available, the packet can have up
2121	3006	* to sgl_size + 2. one for the meta descriptor and one for header
2122	3007	* (if the header is larger than tx_max_header_size).
2123	3008	*/
2124		- if (unlikely(ena_com_sq_empty_space(tx_ring->ena_com_io_sq) <
2125		- (tx_ring->sgl_size + 2))) {
	3009	+ if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
	3010	+ tx_ring->sgl_size + 2))) {
2126	3011	netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n",
2127	3012	__func__, qid);
2128	3013
..	..	@@ -2141,8 +3026,8 @@
2141	3026	*/
2142	3027	smp_mb();
2143	3028
2144		- if (ena_com_sq_empty_space(tx_ring->ena_com_io_sq)
2145		- > ENA_TX_WAKEUP_THRESH) {
	3029	+ if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
	3030	+ ENA_TX_WAKEUP_THRESH)) {
2146	3031	netif_tx_wake_queue(txq);
2147	3032	u64_stats_update_begin(&tx_ring->syncp);
2148	3033	tx_ring->tx_stats.queue_wakeup++;
..	..	@@ -2150,7 +3035,7 @@
2150	3035	}
2151	3036	}
2152	3037
2153		- if (netif_xmit_stopped(txq) \|\| !skb->xmit_more) {
	3038	+ if (netif_xmit_stopped(txq) \|\| !netdev_xmit_more()) {
2154	3039	/* trigger the dma engine. ena_com_write_sq_doorbell()
2155	3040	* has a mb
2156	3041	*/
..	..	@@ -2162,42 +3047,17 @@
2162	3047
2163	3048	return NETDEV_TX_OK;
2164	3049
2165		-error_report_dma_error:
2166		- u64_stats_update_begin(&tx_ring->syncp);
2167		- tx_ring->tx_stats.dma_mapping_err++;
2168		- u64_stats_update_end(&tx_ring->syncp);
2169		- netdev_warn(adapter->netdev, "failed to map skb\n");
2170		-
	3050	+error_unmap_dma:
	3051	+ ena_unmap_tx_buff(tx_ring, tx_info);
2171	3052	tx_info->skb = NULL;
2172	3053
2173		-error_unmap_dma:
2174		- if (i >= 0) {
2175		- /* save value of frag that failed */
2176		- last_frag = i;
2177		-
2178		- /* start back at beginning and unmap skb */
2179		- tx_info->skb = NULL;
2180		- ena_buf = tx_info->bufs;
2181		- dma_unmap_single(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
2182		- dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
2183		-
2184		- /* unmap remaining mapped pages */
2185		- for (i = 0; i < last_frag; i++) {
2186		- ena_buf++;
2187		- dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
2188		- dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
2189		- }
2190		- }
2191		-
2192	3054	error_drop_packet:
2193		-
2194	3055	dev_kfree_skb(skb);
2195	3056	return NETDEV_TX_OK;
2196	3057	}
2197	3058
2198	3059	static u16 ena_select_queue(struct net_device dev, struct sk_buff skb,
2199		- struct net_device *sb_dev,
2200		- select_queue_fallback_t fallback)
	3060	+ struct net_device *sb_dev)
2201	3061	{
2202	3062	u16 qid;
2203	3063	/* we suspect that this is good for in--kernel network services that
..	..	@@ -2207,25 +3067,27 @@
2207	3067	if (skb_rx_queue_recorded(skb))
2208	3068	qid = skb_get_rx_queue(skb);
2209	3069	else
2210		- qid = fallback(dev, skb, NULL);
	3070	+ qid = netdev_pick_tx(dev, skb, NULL);
2211	3071
2212	3072	return qid;
2213	3073	}
2214	3074
2215		-static void ena_config_host_info(struct ena_com_dev *ena_dev)
	3075	+static void ena_config_host_info(struct ena_com_dev ena_dev, struct pci_dev pdev)
2216	3076	{
	3077	+ struct device *dev = &pdev->dev;
2217	3078	struct ena_admin_host_info *host_info;
2218	3079	int rc;
2219	3080
2220	3081	/* Allocate only the host info */
2221	3082	rc = ena_com_allocate_host_info(ena_dev);
2222	3083	if (rc) {
2223		- pr_err("Cannot allocate host info\n");
	3084	+ dev_err(dev, "Cannot allocate host info\n");
2224	3085	return;
2225	3086	}
2226	3087
2227	3088	host_info = ena_dev->host_attr.host_info;
2228	3089
	3090	+ host_info->bdf = (pdev->bus->number << 8) \| pdev->devfn;
2229	3091	host_info->os_type = ENA_ADMIN_OS_LINUX;
2230	3092	host_info->kernel_ver = LINUX_VERSION_CODE;
2231	3093	strlcpy(host_info->kernel_ver_str, utsname()->version,
..	..	@@ -2234,16 +3096,24 @@
2234	3096	strncpy(host_info->os_dist_str, utsname()->release,
2235	3097	sizeof(host_info->os_dist_str) - 1);
2236	3098	host_info->driver_version =
2237		- (DRV_MODULE_VER_MAJOR) \|
2238		- (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) \|
2239		- (DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);
	3099	+ (DRV_MODULE_GEN_MAJOR) \|
	3100	+ (DRV_MODULE_GEN_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) \|
	3101	+ (DRV_MODULE_GEN_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) \|
	3102	+ ("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT);
	3103	+ host_info->num_cpus = num_online_cpus();
	3104	+
	3105	+ host_info->driver_supported_features =
	3106	+ ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK \|
	3107	+ ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK \|
	3108	+ ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK \|
	3109	+ ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK;
2240	3110
2241	3111	rc = ena_com_set_host_attributes(ena_dev);
2242	3112	if (rc) {
2243	3113	if (rc == -EOPNOTSUPP)
2244		- pr_warn("Cannot set host attributes\n");
	3114	+ dev_warn(dev, "Cannot set host attributes\n");
2245	3115	else
2246		- pr_err("Cannot set host attributes\n");
	3116	+ dev_err(dev, "Cannot set host attributes\n");
2247	3117
2248	3118	goto err;
2249	3119	}
..	..	@@ -2271,7 +3141,8 @@
2271	3141
2272	3142	rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size);
2273	3143	if (rc) {
2274		- pr_err("Cannot allocate debug area\n");
	3144	+ netif_err(adapter, drv, adapter->netdev,
	3145	+ "Cannot allocate debug area\n");
2275	3146	return;
2276	3147	}
2277	3148
..	..	@@ -2291,6 +3162,19 @@
2291	3162	ena_com_delete_debug_area(adapter->ena_dev);
2292	3163	}
2293	3164
	3165	+int ena_update_hw_stats(struct ena_adapter *adapter)
	3166	+{
	3167	+ int rc = 0;
	3168	+
	3169	+ rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_stats);
	3170	+ if (rc) {
	3171	+ dev_info_once(&adapter->pdev->dev, "Failed to get ENI stats\n");
	3172	+ return rc;
	3173	+ }
	3174	+
	3175	+ return 0;
	3176	+}
	3177	+
2294	3178	static void ena_get_stats64(struct net_device *netdev,
2295	3179	struct rtnl_link_stats64 *stats)
2296	3180	{
..	..	@@ -2298,12 +3182,13 @@
2298	3182	struct ena_ring rx_ring, tx_ring;
2299	3183	unsigned int start;
2300	3184	u64 rx_drops;
	3185	+ u64 tx_drops;
2301	3186	int i;
2302	3187
2303	3188	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
2304	3189	return;
2305	3190
2306		- for (i = 0; i < adapter->num_queues; i++) {
	3191	+ for (i = 0; i < adapter->num_io_queues; i++) {
2307	3192	u64 bytes, packets;
2308	3193
2309	3194	tx_ring = &adapter->tx_ring[i];
..	..	@@ -2332,9 +3217,11 @@
2332	3217	do {
2333	3218	start = u64_stats_fetch_begin_irq(&adapter->syncp);
2334	3219	rx_drops = adapter->dev_stats.rx_drops;
	3220	+ tx_drops = adapter->dev_stats.tx_drops;
2335	3221	} while (u64_stats_fetch_retry_irq(&adapter->syncp, start));
2336	3222
2337	3223	stats->rx_dropped = rx_drops;
	3224	+ stats->tx_dropped = tx_drops;
2338	3225
2339	3226	stats->multicast = 0;
2340	3227	stats->collisions = 0;
..	..	@@ -2360,6 +3247,7 @@
2360	3247	.ndo_change_mtu = ena_change_mtu,
2361	3248	.ndo_set_mac_address = NULL,
2362	3249	.ndo_validate_addr = eth_validate_addr,
	3250	+ .ndo_bpf = ena_xdp,
2363	3251	};
2364	3252
2365	3253	static int ena_device_validate_params(struct ena_adapter *adapter,
..	..	@@ -2376,13 +3264,6 @@
2376	3264	return -EINVAL;
2377	3265	}
2378	3266
2379		- if ((get_feat_ctx->max_queues.max_cq_num < adapter->num_queues) \|\|
2380		- (get_feat_ctx->max_queues.max_sq_num < adapter->num_queues)) {
2381		- netif_err(adapter, drv, netdev,
2382		- "Error, device doesn't support enough queues\n");
2383		- return -EINVAL;
2384		- }
2385		-
2386	3267	if (get_feat_ctx->dev_attr.max_mtu < netdev->mtu) {
2387	3268	netif_err(adapter, drv, netdev,
2388	3269	"Error, device max mtu is smaller than netdev MTU\n");
..	..	@@ -2392,10 +3273,71 @@
2392	3273	return 0;
2393	3274	}
2394	3275
	3276	+static void set_default_llq_configurations(struct ena_llq_configurations *llq_config)
	3277	+{
	3278	+ llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
	3279	+ llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
	3280	+ llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
	3281	+ llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
	3282	+ llq_config->llq_ring_entry_size_value = 128;
	3283	+}
	3284	+
	3285	+static int ena_set_queues_placement_policy(struct pci_dev *pdev,
	3286	+ struct ena_com_dev *ena_dev,
	3287	+ struct ena_admin_feature_llq_desc *llq,
	3288	+ struct ena_llq_configurations *llq_default_configurations)
	3289	+{
	3290	+ int rc;
	3291	+ u32 llq_feature_mask;
	3292	+
	3293	+ llq_feature_mask = 1 << ENA_ADMIN_LLQ;
	3294	+ if (!(ena_dev->supported_features & llq_feature_mask)) {
	3295	+ dev_err(&pdev->dev,
	3296	+ "LLQ is not supported Fallback to host mode policy.\n");
	3297	+ ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
	3298	+ return 0;
	3299	+ }
	3300	+
	3301	+ rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
	3302	+ if (unlikely(rc)) {
	3303	+ dev_err(&pdev->dev,
	3304	+ "Failed to configure the device mode. Fallback to host mode policy.\n");
	3305	+ ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
	3306	+ }
	3307	+
	3308	+ return 0;
	3309	+}
	3310	+
	3311	+static int ena_map_llq_mem_bar(struct pci_dev pdev, struct ena_com_dev ena_dev,
	3312	+ int bars)
	3313	+{
	3314	+ bool has_mem_bar = !!(bars & BIT(ENA_MEM_BAR));
	3315	+
	3316	+ if (!has_mem_bar) {
	3317	+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
	3318	+ dev_err(&pdev->dev,
	3319	+ "ENA device does not expose LLQ bar. Fallback to host mode policy.\n");
	3320	+ ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
	3321	+ }
	3322	+
	3323	+ return 0;
	3324	+ }
	3325	+
	3326	+ ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
	3327	+ pci_resource_start(pdev, ENA_MEM_BAR),
	3328	+ pci_resource_len(pdev, ENA_MEM_BAR));
	3329	+
	3330	+ if (!ena_dev->mem_bar)
	3331	+ return -EFAULT;
	3332	+
	3333	+ return 0;
	3334	+}
	3335	+
2395	3336	static int ena_device_init(struct ena_com_dev ena_dev, struct pci_dev pdev,
2396	3337	struct ena_com_dev_get_features_ctx *get_feat_ctx,
2397	3338	bool *wd_state)
2398	3339	{
	3340	+ struct ena_llq_configurations llq_config;
2399	3341	struct device *dev = &pdev->dev;
2400	3342	bool readless_supported;
2401	3343	u32 aenq_groups;
..	..	@@ -2404,7 +3346,7 @@
2404	3346
2405	3347	rc = ena_com_mmio_reg_read_request_init(ena_dev);
2406	3348	if (rc) {
2407		- dev_err(dev, "failed to init mmio read less\n");
	3349	+ dev_err(dev, "Failed to init mmio read less\n");
2408	3350	return rc;
2409	3351	}
2410	3352
..	..	@@ -2422,7 +3364,7 @@
2422	3364
2423	3365	rc = ena_com_validate_version(ena_dev);
2424	3366	if (rc) {
2425		- dev_err(dev, "device version is too low\n");
	3367	+ dev_err(dev, "Device version is too low\n");
2426	3368	goto err_mmio_read_less;
2427	3369	}
2428	3370
..	..	@@ -2440,7 +3382,7 @@
2440	3382	}
2441	3383
2442	3384	/* ENA admin level init */
2443		- rc = ena_com_admin_init(ena_dev, &aenq_handlers, true);
	3385	+ rc = ena_com_admin_init(ena_dev, &aenq_handlers);
2444	3386	if (rc) {
2445	3387	dev_err(dev,
2446	3388	"Can not initialize ena admin queue with device\n");
..	..	@@ -2453,7 +3395,7 @@
2453	3395	*/
2454	3396	ena_com_set_admin_polling_mode(ena_dev, true);
2455	3397
2456		- ena_config_host_info(ena_dev);
	3398	+ ena_config_host_info(ena_dev, pdev);
2457	3399
2458	3400	/* Get Device Attributes*/
2459	3401	rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
..	..	@@ -2479,6 +3421,15 @@
2479	3421
2480	3422	*wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
2481	3423
	3424	+ set_default_llq_configurations(&llq_config);
	3425	+
	3426	+ rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq,
	3427	+ &llq_config);
	3428	+ if (rc) {
	3429	+ dev_err(dev, "ENA device init failed\n");
	3430	+ goto err_admin_init;
	3431	+ }
	3432	+
2482	3433	return 0;
2483	3434
2484	3435	err_admin_init:
..	..	@@ -2490,14 +3441,13 @@
2490	3441	return rc;
2491	3442	}
2492	3443
2493		-static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter,
2494		- int io_vectors)
	3444	+static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter)
2495	3445	{
2496	3446	struct ena_com_dev *ena_dev = adapter->ena_dev;
2497	3447	struct device *dev = &adapter->pdev->dev;
2498	3448	int rc;
2499	3449
2500		- rc = ena_enable_msix(adapter, io_vectors);
	3450	+ rc = ena_enable_msix(adapter);
2501	3451	if (rc) {
2502	3452	dev_err(dev, "Can not reserve msix vectors\n");
2503	3453	return rc;
..	..	@@ -2538,17 +3488,14 @@
2538	3488
2539	3489	dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2540	3490	adapter->dev_up_before_reset = dev_up;
2541		-
2542	3491	if (!graceful)
2543	3492	ena_com_set_admin_running_state(ena_dev, false);
2544	3493
2545	3494	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
2546	3495	ena_down(adapter);
2547	3496
2548		- /* Before releasing the ENA resources, a device reset is required.
2549		- * (to prevent the device from accessing them).
2550		- * In case the reset flag is set and the device is up, ena_down()
2551		- * already perform the reset, so it can be skipped.
	3497	+ /* Stop the device from sending AENQ events (in case reset flag is set
	3498	+ * and device is up, ena_down() already reset the device.
2552	3499	*/
2553	3500	if (!(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags) && dev_up))
2554	3501	ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
..	..	@@ -2565,6 +3512,7 @@
2565	3512
2566	3513	ena_com_mmio_reg_read_request_destroy(ena_dev);
2567	3514
	3515	+ /* return reset reason to default value */
2568	3516	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
2569	3517
2570	3518	clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
..	..	@@ -2593,8 +3541,7 @@
2593	3541	goto err_device_destroy;
2594	3542	}
2595	3543
2596		- rc = ena_enable_msix_and_set_admin_interrupts(adapter,
2597		- adapter->num_queues);
	3544	+ rc = ena_enable_msix_and_set_admin_interrupts(adapter);
2598	3545	if (rc) {
2599	3546	dev_err(&pdev->dev, "Enable MSI-X failed\n");
2600	3547	goto err_device_destroy;
..	..	@@ -2615,6 +3562,8 @@
2615	3562	netif_carrier_on(adapter->netdev);
2616	3563
2617	3564	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
	3565	+ adapter->last_keep_alive_jiffies = jiffies;
	3566	+
2618	3567	dev_err(&pdev->dev, "Device reset completed successfully\n");
2619	3568
2620	3569	return rc;
..	..	@@ -2740,7 +3689,9 @@
2740	3689	struct ena_ring *tx_ring;
2741	3690	struct ena_ring *rx_ring;
2742	3691	int i, budget, rc;
	3692	+ int io_queue_count;
2743	3693
	3694	+ io_queue_count = adapter->xdp_num_queues + adapter->num_io_queues;
2744	3695	/* Make sure the driver doesn't turn the device in other process */
2745	3696	smp_rmb();
2746	3697
..	..	@@ -2755,7 +3706,7 @@
2755	3706
2756	3707	budget = ENA_MONITORED_TX_QUEUES;
2757	3708
2758		- for (i = adapter->last_monitored_tx_qid; i < adapter->num_queues; i++) {
	3709	+ for (i = adapter->last_monitored_tx_qid; i < io_queue_count; i++) {
2759	3710	tx_ring = &adapter->tx_ring[i];
2760	3711	rx_ring = &adapter->rx_ring[i];
2761	3712
..	..	@@ -2763,7 +3714,8 @@
2763	3714	if (unlikely(rc))
2764	3715	return;
2765	3716
2766		- rc = check_for_rx_interrupt_queue(adapter, rx_ring);
	3717	+ rc = !ENA_IS_XDP_INDEX(adapter, i) ?
	3718	+ check_for_rx_interrupt_queue(adapter, rx_ring) : 0;
2767	3719	if (unlikely(rc))
2768	3720	return;
2769	3721
..	..	@@ -2772,7 +3724,7 @@
2772	3724	break;
2773	3725	}
2774	3726
2775		- adapter->last_monitored_tx_qid = i % adapter->num_queues;
	3727	+ adapter->last_monitored_tx_qid = i % io_queue_count;
2776	3728	}
2777	3729
2778	3730	/* trigger napi schedule after 2 consecutive detections */
..	..	@@ -2802,11 +3754,10 @@
2802	3754	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
2803	3755	return;
2804	3756
2805		- for (i = 0; i < adapter->num_queues; i++) {
	3757	+ for (i = 0; i < adapter->num_io_queues; i++) {
2806	3758	rx_ring = &adapter->rx_ring[i];
2807	3759
2808		- refill_required =
2809		- ena_com_sq_empty_space(rx_ring->ena_com_io_sq);
	3760	+ refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
2810	3761	if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
2811	3762	rx_ring->empty_rx_queue++;
2812	3763
..	..	@@ -2816,7 +3767,7 @@
2816	3767	u64_stats_update_end(&rx_ring->syncp);
2817	3768
2818	3769	netif_err(adapter, drv, adapter->netdev,
2819		- "trigger refill for ring %d\n", i);
	3770	+ "Trigger refill for ring %d\n", i);
2820	3771
2821	3772	napi_schedule(rx_ring->napi);
2822	3773	rx_ring->empty_rx_queue = 0;
..	..	@@ -2944,54 +3895,40 @@
2944	3895	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
2945	3896	}
2946	3897
2947		-static int ena_calc_io_queue_num(struct pci_dev *pdev,
2948		- struct ena_com_dev *ena_dev,
2949		- struct ena_com_dev_get_features_ctx *get_feat_ctx)
	3898	+static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev,
	3899	+ struct ena_com_dev *ena_dev,
	3900	+ struct ena_com_dev_get_features_ctx *get_feat_ctx)
2950	3901	{
2951		- int io_sq_num, io_queue_num;
	3902	+ u32 io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
2952	3903
2953		- /* In case of LLQ use the llq number in the get feature cmd */
2954		- if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2955		- io_sq_num = get_feat_ctx->max_queues.max_llq_num;
	3904	+ if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
	3905	+ struct ena_admin_queue_ext_feature_fields *max_queue_ext =
	3906	+ &get_feat_ctx->max_queue_ext.max_queue_ext;
	3907	+ io_rx_num = min_t(u32, max_queue_ext->max_rx_sq_num,
	3908	+ max_queue_ext->max_rx_cq_num);
2956	3909
2957		- if (io_sq_num == 0) {
2958		- dev_err(&pdev->dev,
2959		- "Trying to use LLQ but llq_num is 0. Fall back into regular queues\n");
2960		-
2961		- ena_dev->tx_mem_queue_type =
2962		- ENA_ADMIN_PLACEMENT_POLICY_HOST;
2963		- io_sq_num = get_feat_ctx->max_queues.max_sq_num;
2964		- }
	3910	+ io_tx_sq_num = max_queue_ext->max_tx_sq_num;
	3911	+ io_tx_cq_num = max_queue_ext->max_tx_cq_num;
2965	3912	} else {
2966		- io_sq_num = get_feat_ctx->max_queues.max_sq_num;
	3913	+ struct ena_admin_queue_feature_desc *max_queues =
	3914	+ &get_feat_ctx->max_queues;
	3915	+ io_tx_sq_num = max_queues->max_sq_num;
	3916	+ io_tx_cq_num = max_queues->max_cq_num;
	3917	+ io_rx_num = min_t(u32, io_tx_sq_num, io_tx_cq_num);
2967	3918	}
2968	3919
2969		- io_queue_num = min_t(int, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES);
2970		- io_queue_num = min_t(int, io_queue_num, io_sq_num);
2971		- io_queue_num = min_t(int, io_queue_num,
2972		- get_feat_ctx->max_queues.max_cq_num);
	3920	+ /* In case of LLQ use the llq fields for the tx SQ/CQ */
	3921	+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
	3922	+ io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
	3923	+
	3924	+ max_num_io_queues = min_t(u32, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES);
	3925	+ max_num_io_queues = min_t(u32, max_num_io_queues, io_rx_num);
	3926	+ max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_sq_num);
	3927	+ max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_cq_num);
2973	3928	/* 1 IRQ for for mgmnt and 1 IRQs for each IO direction */
2974		- io_queue_num = min_t(int, io_queue_num, pci_msix_vec_count(pdev) - 1);
2975		- if (unlikely(!io_queue_num)) {
2976		- dev_err(&pdev->dev, "The device doesn't have io queues\n");
2977		- return -EFAULT;
2978		- }
	3929	+ max_num_io_queues = min_t(u32, max_num_io_queues, pci_msix_vec_count(pdev) - 1);
2979	3930
2980		- return io_queue_num;
2981		-}
2982		-
2983		-static void ena_set_push_mode(struct pci_dev pdev, struct ena_com_dev ena_dev,
2984		- struct ena_com_dev_get_features_ctx *get_feat_ctx)
2985		-{
2986		- bool has_mem_bar;
2987		-
2988		- has_mem_bar = pci_select_bars(pdev, IORESOURCE_MEM) & BIT(ENA_MEM_BAR);
2989		-
2990		- /* Enable push mode if device supports LLQ */
2991		- if (has_mem_bar && (get_feat_ctx->max_queues.max_llq_num > 0))
2992		- ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV;
2993		- else
2994		- ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
	3931	+ return max_num_io_queues;
2995	3932	}
2996	3933
2997	3934	static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,
..	..	@@ -3071,7 +4008,7 @@
3071	4008	}
3072	4009
3073	4010	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
3074		- val = ethtool_rxfh_indir_default(i, adapter->num_queues);
	4011	+ val = ethtool_rxfh_indir_default(i, adapter->num_io_queues);
3075	4012	rc = ena_com_indirect_table_fill_entry(ena_dev, i,
3076	4013	ENA_IO_RXQ_IDX(val));
3077	4014	if (unlikely(rc && (rc != -EOPNOTSUPP))) {
..	..	@@ -3080,7 +4017,7 @@
3080	4017	}
3081	4018	}
3082	4019
3083		- rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL,
	4020	+ rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, NULL,
3084	4021	ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
3085	4022	if (unlikely(rc && (rc != -EOPNOTSUPP))) {
3086	4023	dev_err(dev, "Cannot fill hash function\n");
..	..	@@ -3109,36 +4046,71 @@
3109	4046	pci_release_selected_regions(pdev, release_bars);
3110	4047	}
3111	4048
3112		-static int ena_calc_queue_size(struct pci_dev *pdev,
3113		- struct ena_com_dev *ena_dev,
3114		- u16 *max_tx_sgl_size,
3115		- u16 *max_rx_sgl_size,
3116		- struct ena_com_dev_get_features_ctx *get_feat_ctx)
	4049	+
	4050	+static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx)
3117	4051	{
3118		- u32 queue_size = ENA_DEFAULT_RING_SIZE;
	4052	+ struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq;
	4053	+ struct ena_com_dev *ena_dev = ctx->ena_dev;
	4054	+ u32 tx_queue_size = ENA_DEFAULT_RING_SIZE;
	4055	+ u32 rx_queue_size = ENA_DEFAULT_RING_SIZE;
	4056	+ u32 max_tx_queue_size;
	4057	+ u32 max_rx_queue_size;
3119	4058
3120		- queue_size = min_t(u32, queue_size,
3121		- get_feat_ctx->max_queues.max_cq_depth);
3122		- queue_size = min_t(u32, queue_size,
3123		- get_feat_ctx->max_queues.max_sq_depth);
	4059	+ if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
	4060	+ struct ena_admin_queue_ext_feature_fields *max_queue_ext =
	4061	+ &ctx->get_feat_ctx->max_queue_ext.max_queue_ext;
	4062	+ max_rx_queue_size = min_t(u32, max_queue_ext->max_rx_cq_depth,
	4063	+ max_queue_ext->max_rx_sq_depth);
	4064	+ max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
3124	4065
3125		- if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
3126		- queue_size = min_t(u32, queue_size,
3127		- get_feat_ctx->max_queues.max_llq_depth);
	4066	+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
	4067	+ max_tx_queue_size = min_t(u32, max_tx_queue_size,
	4068	+ llq->max_llq_depth);
	4069	+ else
	4070	+ max_tx_queue_size = min_t(u32, max_tx_queue_size,
	4071	+ max_queue_ext->max_tx_sq_depth);
3128	4072
3129		- queue_size = rounddown_pow_of_two(queue_size);
	4073	+ ctx->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
	4074	+ max_queue_ext->max_per_packet_tx_descs);
	4075	+ ctx->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
	4076	+ max_queue_ext->max_per_packet_rx_descs);
	4077	+ } else {
	4078	+ struct ena_admin_queue_feature_desc *max_queues =
	4079	+ &ctx->get_feat_ctx->max_queues;
	4080	+ max_rx_queue_size = min_t(u32, max_queues->max_cq_depth,
	4081	+ max_queues->max_sq_depth);
	4082	+ max_tx_queue_size = max_queues->max_cq_depth;
3130	4083
3131		- if (unlikely(!queue_size)) {
3132		- dev_err(&pdev->dev, "Invalid queue size\n");
3133		- return -EFAULT;
	4084	+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
	4085	+ max_tx_queue_size = min_t(u32, max_tx_queue_size,
	4086	+ llq->max_llq_depth);
	4087	+ else
	4088	+ max_tx_queue_size = min_t(u32, max_tx_queue_size,
	4089	+ max_queues->max_sq_depth);
	4090	+
	4091	+ ctx->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
	4092	+ max_queues->max_packet_tx_descs);
	4093	+ ctx->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
	4094	+ max_queues->max_packet_rx_descs);
3134	4095	}
3135	4096
3136		- *max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
3137		- get_feat_ctx->max_queues.max_packet_tx_descs);
3138		- *max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
3139		- get_feat_ctx->max_queues.max_packet_rx_descs);
	4097	+ max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size);
	4098	+ max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size);
3140	4099
3141		- return queue_size;
	4100	+ tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
	4101	+ max_tx_queue_size);
	4102	+ rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
	4103	+ max_rx_queue_size);
	4104	+
	4105	+ tx_queue_size = rounddown_pow_of_two(tx_queue_size);
	4106	+ rx_queue_size = rounddown_pow_of_two(rx_queue_size);
	4107	+
	4108	+ ctx->max_tx_queue_size = max_tx_queue_size;
	4109	+ ctx->max_rx_queue_size = max_rx_queue_size;
	4110	+ ctx->tx_queue_size = tx_queue_size;
	4111	+ ctx->rx_queue_size = rx_queue_size;
	4112	+
	4113	+ return 0;
3142	4114	}
3143	4115
3144	4116	/* ena_probe - Device Initialization Routine
..	..	@@ -3153,22 +4125,17 @@
3153	4125	*/
3154	4126	static int ena_probe(struct pci_dev pdev, const struct pci_device_id ent)
3155	4127	{
	4128	+ struct ena_calc_queue_size_ctx calc_queue_ctx = {};
3156	4129	struct ena_com_dev_get_features_ctx get_feat_ctx;
3157		- static int version_printed;
3158		- struct net_device *netdev;
3159		- struct ena_adapter *adapter;
3160	4130	struct ena_com_dev *ena_dev = NULL;
	4131	+ struct ena_adapter *adapter;
	4132	+ struct net_device *netdev;
3161	4133	static int adapters_found;
3162		- int io_queue_num, bars, rc;
3163		- int queue_size;
3164		- u16 tx_sgl_size = 0;
3165		- u16 rx_sgl_size = 0;
	4134	+ u32 max_num_io_queues;
3166	4135	bool wd_state;
	4136	+ int bars, rc;
3167	4137
3168	4138	dev_dbg(&pdev->dev, "%s\n", __func__);
3169		-
3170		- if (version_printed++ == 0)
3171		- dev_info(&pdev->dev, "%s", version);
3172	4139
3173	4140	rc = pci_enable_device_mem(pdev);
3174	4141	if (rc) {
..	..	@@ -3202,50 +4169,48 @@
3202	4169	pci_resource_start(pdev, ENA_REG_BAR),
3203	4170	pci_resource_len(pdev, ENA_REG_BAR));
3204	4171	if (!ena_dev->reg_bar) {
3205		- dev_err(&pdev->dev, "failed to remap regs bar\n");
	4172	+ dev_err(&pdev->dev, "Failed to remap regs bar\n");
3206	4173	rc = -EFAULT;
3207	4174	goto err_free_region;
3208	4175	}
	4176	+
	4177	+ ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US;
3209	4178
3210	4179	ena_dev->dmadev = &pdev->dev;
3211	4180
3212	4181	rc = ena_device_init(ena_dev, pdev, &get_feat_ctx, &wd_state);
3213	4182	if (rc) {
3214		- dev_err(&pdev->dev, "ena device init failed\n");
	4183	+ dev_err(&pdev->dev, "ENA device init failed\n");
3215	4184	if (rc == -ETIME)
3216	4185	rc = -EPROBE_DEFER;
3217	4186	goto err_free_region;
3218	4187	}
3219	4188
3220		- ena_set_push_mode(pdev, ena_dev, &get_feat_ctx);
3221		-
3222		- if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
3223		- ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
3224		- pci_resource_start(pdev, ENA_MEM_BAR),
3225		- pci_resource_len(pdev, ENA_MEM_BAR));
3226		- if (!ena_dev->mem_bar) {
3227		- rc = -EFAULT;
3228		- goto err_device_destroy;
3229		- }
	4189	+ rc = ena_map_llq_mem_bar(pdev, ena_dev, bars);
	4190	+ if (rc) {
	4191	+ dev_err(&pdev->dev, "ENA llq bar mapping failed\n");
	4192	+ goto err_free_ena_dev;
3230	4193	}
3231	4194
3232		- /* initial Tx interrupt delay, Assumes 1 usec granularity.
3233		- * Updated during device initialization with the real granularity
3234		- */
	4195	+ calc_queue_ctx.ena_dev = ena_dev;
	4196	+ calc_queue_ctx.get_feat_ctx = &get_feat_ctx;
	4197	+ calc_queue_ctx.pdev = pdev;
	4198	+
	4199	+ /* Initial TX and RX interrupt delay. Assumes 1 usec granularity.
	4200	+ * Updated during device initialization with the real granularity
	4201	+ */
3235	4202	ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS;
3236		- io_queue_num = ena_calc_io_queue_num(pdev, ena_dev, &get_feat_ctx);
3237		- queue_size = ena_calc_queue_size(pdev, ena_dev, &tx_sgl_size,
3238		- &rx_sgl_size, &get_feat_ctx);
3239		- if ((queue_size <= 0) \|\| (io_queue_num <= 0)) {
	4203	+ ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS;
	4204	+ ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
	4205	+ max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, &get_feat_ctx);
	4206	+ rc = ena_calc_io_queue_size(&calc_queue_ctx);
	4207	+ if (rc \|\| !max_num_io_queues) {
3240	4208	rc = -EFAULT;
3241	4209	goto err_device_destroy;
3242	4210	}
3243	4211
3244		- dev_info(&pdev->dev, "creating %d io queues. queue size: %d\n",
3245		- io_queue_num, queue_size);
3246		-
3247	4212	/* dev zeroed in init_etherdev */
3248		- netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), io_queue_num);
	4213	+ netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), max_num_io_queues);
3249	4214	if (!netdev) {
3250	4215	dev_err(&pdev->dev, "alloc_etherdev_mq failed\n");
3251	4216	rc = -ENOMEM;
..	..	@@ -3266,16 +4231,26 @@
3266	4231	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3267	4232	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3268	4233
3269		- adapter->tx_ring_size = queue_size;
3270		- adapter->rx_ring_size = queue_size;
	4234	+ adapter->requested_tx_ring_size = calc_queue_ctx.tx_queue_size;
	4235	+ adapter->requested_rx_ring_size = calc_queue_ctx.rx_queue_size;
	4236	+ adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size;
	4237	+ adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size;
	4238	+ adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size;
	4239	+ adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size;
3271	4240
3272		- adapter->max_tx_sgl_size = tx_sgl_size;
3273		- adapter->max_rx_sgl_size = rx_sgl_size;
3274		-
3275		- adapter->num_queues = io_queue_num;
	4241	+ adapter->num_io_queues = max_num_io_queues;
	4242	+ adapter->max_num_io_queues = max_num_io_queues;
3276	4243	adapter->last_monitored_tx_qid = 0;
3277	4244
	4245	+ adapter->xdp_first_ring = 0;
	4246	+ adapter->xdp_num_queues = 0;
	4247	+
3278	4248	adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK;
	4249	+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
	4250	+ adapter->disable_meta_caching =
	4251	+ !!(get_feat_ctx.llq.accel_mode.u.get.supported_flags &
	4252	+ BIT(ENA_ADMIN_DISABLE_META_CACHING));
	4253	+
3279	4254	adapter->wd_state = wd_state;
3280	4255
3281	4256	snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found);
..	..	@@ -3286,7 +4261,10 @@
3286	4261	"Failed to query interrupt moderation feature\n");
3287	4262	goto err_netdev_destroy;
3288	4263	}
3289		- ena_init_io_rings(adapter);
	4264	+ ena_init_io_rings(adapter,
	4265	+ 0,
	4266	+ adapter->xdp_num_queues +
	4267	+ adapter->num_io_queues);
3290	4268
3291	4269	netdev->netdev_ops = &ena_netdev_ops;
3292	4270	netdev->watchdog_timeo = TX_TIMEOUT;
..	..	@@ -3296,7 +4274,7 @@
3296	4274
3297	4275	u64_stats_init(&adapter->syncp);
3298	4276
3299		- rc = ena_enable_msix_and_set_admin_interrupts(adapter, io_queue_num);
	4277	+ rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3300	4278	if (rc) {
3301	4279	dev_err(&pdev->dev,
3302	4280	"Failed to enable and set the admin interrupts\n");
..	..	@@ -3309,6 +4287,11 @@
3309	4287	}
3310	4288
3311	4289	ena_config_debug_area(adapter);
	4290	+
	4291	+ if (!ena_update_hw_stats(adapter))
	4292	+ adapter->eni_stats_supported = true;
	4293	+ else
	4294	+ adapter->eni_stats_supported = false;
3312	4295
3313	4296	memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len);
3314	4297
..	..	@@ -3332,9 +4315,10 @@
3332	4315	timer_setup(&adapter->timer_service, ena_timer_service, 0);
3333	4316	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3334	4317
3335		- dev_info(&pdev->dev, "%s found at mem %lx, mac addr %pM Queues %d\n",
	4318	+ dev_info(&pdev->dev,
	4319	+ "%s found at mem %lx, mac addr %pM\n",
3336	4320	DEVICE_NAME, (long)pci_resource_start(pdev, 0),
3337		- netdev->dev_addr, io_queue_num);
	4321	+ netdev->dev_addr);
3338	4322
3339	4323	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3340	4324
..	..	@@ -3347,10 +4331,11 @@
3347	4331	ena_com_rss_destroy(ena_dev);
3348	4332	err_free_msix:
3349	4333	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);
	4334	+ /* stop submitting admin commands on a device that was reset */
	4335	+ ena_com_set_admin_running_state(ena_dev, false);
3350	4336	ena_free_mgmnt_irq(adapter);
3351	4337	ena_disable_msix(adapter);
3352	4338	err_worker_destroy:
3353		- ena_com_destroy_interrupt_moderation(ena_dev);
3354	4339	del_timer(&adapter->timer_service);
3355	4340	err_netdev_destroy:
3356	4341	free_netdev(netdev);
..	..	@@ -3368,13 +4353,15 @@
3368	4353
3369	4354	/*****************************************************************************/
3370	4355
3371		-/* ena_remove - Device Removal Routine
	4356	+/* __ena_shutoff - Helper used in both PCI remove/shutdown routines
3372	4357	* @pdev: PCI device information struct
	4358	+ * @shutdown: Is it a shutdown operation? If false, means it is a removal
3373	4359	*
3374		- * ena_remove is called by the PCI subsystem to alert the driver
3375		- * that it should release a PCI device.
	4360	+ * __ena_shutoff is a helper routine that does the real work on shutdown and
	4361	+ * removal paths; the difference between those paths is with regards to whether
	4362	+ * dettach or unregister the netdevice.
3376	4363	*/
3377		-static void ena_remove(struct pci_dev *pdev)
	4364	+static void __ena_shutoff(struct pci_dev *pdev, bool shutdown)
3378	4365	{
3379	4366	struct ena_adapter *adapter = pci_get_drvdata(pdev);
3380	4367	struct ena_com_dev *ena_dev;
..	..	@@ -3396,19 +4383,18 @@
3396	4383	del_timer_sync(&adapter->timer_service);
3397	4384	cancel_work_sync(&adapter->reset_task);
3398	4385
3399		- unregister_netdev(netdev);
3400		-
3401		- /* If the device is running then we want to make sure the device will be
3402		- * reset to make sure no more events will be issued by the device.
3403		- */
3404		- if (test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
3405		- set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3406		-
3407		- rtnl_lock();
	4386	+ rtnl_lock(); /* lock released inside the below if-else block */
	4387	+ adapter->reset_reason = ENA_REGS_RESET_SHUTDOWN;
3408	4388	ena_destroy_device(adapter, true);
3409		- rtnl_unlock();
3410		-
3411		- free_netdev(netdev);
	4389	+ if (shutdown) {
	4390	+ netif_device_detach(netdev);
	4391	+ dev_close(netdev);
	4392	+ rtnl_unlock();
	4393	+ } else {
	4394	+ rtnl_unlock();
	4395	+ unregister_netdev(netdev);
	4396	+ free_netdev(netdev);
	4397	+ }
3412	4398
3413	4399	ena_com_rss_destroy(ena_dev);
3414	4400
..	..	@@ -3420,18 +4406,39 @@
3420	4406
3421	4407	pci_disable_device(pdev);
3422	4408
3423		- ena_com_destroy_interrupt_moderation(ena_dev);
3424		-
3425	4409	vfree(ena_dev);
3426	4410	}
3427	4411
3428		-#ifdef CONFIG_PM
3429		-/* ena_suspend - PM suspend callback
	4412	+/* ena_remove - Device Removal Routine
3430	4413	* @pdev: PCI device information struct
3431		- * @state:power state
	4414	+ *
	4415	+ * ena_remove is called by the PCI subsystem to alert the driver
	4416	+ * that it should release a PCI device.
3432	4417	*/
3433		-static int ena_suspend(struct pci_dev *pdev, pm_message_t state)
	4418	+
	4419	+static void ena_remove(struct pci_dev *pdev)
3434	4420	{
	4421	+ __ena_shutoff(pdev, false);
	4422	+}
	4423	+
	4424	+/* ena_shutdown - Device Shutdown Routine
	4425	+ * @pdev: PCI device information struct
	4426	+ *
	4427	+ * ena_shutdown is called by the PCI subsystem to alert the driver that
	4428	+ * a shutdown/reboot (or kexec) is happening and device must be disabled.
	4429	+ */
	4430	+
	4431	+static void ena_shutdown(struct pci_dev *pdev)
	4432	+{
	4433	+ __ena_shutoff(pdev, true);
	4434	+}
	4435	+
	4436	+/* ena_suspend - PM suspend callback
	4437	+ * @dev_d: Device information struct
	4438	+ */
	4439	+static int __maybe_unused ena_suspend(struct device *dev_d)
	4440	+{
	4441	+ struct pci_dev *pdev = to_pci_dev(dev_d);
3435	4442	struct ena_adapter *adapter = pci_get_drvdata(pdev);
3436	4443
3437	4444	u64_stats_update_begin(&adapter->syncp);
..	..	@@ -3441,7 +4448,7 @@
3441	4448	rtnl_lock();
3442	4449	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
3443	4450	dev_err(&pdev->dev,
3444		- "ignoring device reset request as the device is being suspended\n");
	4451	+ "Ignoring device reset request as the device is being suspended\n");
3445	4452	clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3446	4453	}
3447	4454	ena_destroy_device(adapter, true);
..	..	@@ -3450,12 +4457,11 @@
3450	4457	}
3451	4458
3452	4459	/* ena_resume - PM resume callback
3453		- * @pdev: PCI device information struct
3454		- *
	4460	+ * @dev_d: Device information struct
3455	4461	*/
3456		-static int ena_resume(struct pci_dev *pdev)
	4462	+static int __maybe_unused ena_resume(struct device *dev_d)
3457	4463	{
3458		- struct ena_adapter *adapter = pci_get_drvdata(pdev);
	4464	+ struct ena_adapter *adapter = dev_get_drvdata(dev_d);
3459	4465	int rc;
3460	4466
3461	4467	u64_stats_update_begin(&adapter->syncp);
..	..	@@ -3467,23 +4473,22 @@
3467	4473	rtnl_unlock();
3468	4474	return rc;
3469	4475	}
3470		-#endif
	4476	+
	4477	+static SIMPLE_DEV_PM_OPS(ena_pm_ops, ena_suspend, ena_resume);
3471	4478
3472	4479	static struct pci_driver ena_pci_driver = {
3473	4480	.name = DRV_MODULE_NAME,
3474	4481	.id_table = ena_pci_tbl,
3475	4482	.probe = ena_probe,
3476	4483	.remove = ena_remove,
3477		-#ifdef CONFIG_PM
3478		- .suspend = ena_suspend,
3479		- .resume = ena_resume,
3480		-#endif
	4484	+ .shutdown = ena_shutdown,
	4485	+ .driver.pm = &ena_pm_ops,
3481	4486	.sriov_configure = pci_sriov_configure_simple,
3482	4487	};
3483	4488
3484	4489	static int __init ena_init(void)
3485	4490	{
3486		- pr_info("%s", version);
	4491	+ int ret;
3487	4492
3488	4493	ena_wq = create_singlethread_workqueue(DRV_MODULE_NAME);
3489	4494	if (!ena_wq) {
..	..	@@ -3491,7 +4496,11 @@
3491	4496	return -ENOMEM;
3492	4497	}
3493	4498
3494		- return pci_register_driver(&ena_pci_driver);
	4499	+ ret = pci_register_driver(&ena_pci_driver);
	4500	+ if (ret)
	4501	+ destroy_workqueue(ena_wq);
	4502	+
	4503	+ return ret;
3495	4504	}
3496	4505
3497	4506	static void __exit ena_cleanup(void)
..	..	@@ -3520,7 +4529,7 @@
3520	4529	ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
3521	4530
3522	4531	if (status) {
3523		- netdev_dbg(adapter->netdev, "%s\n", __func__);
	4532	+ netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);
3524	4533	set_bit(ENA_FLAG_LINK_UP, &adapter->flags);
3525	4534	if (!test_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags))
3526	4535	netif_carrier_on(adapter->netdev);
..	..	@@ -3536,17 +4545,20 @@
3536	4545	struct ena_adapter adapter = (struct ena_adapter )adapter_data;
3537	4546	struct ena_admin_aenq_keep_alive_desc *desc;
3538	4547	u64 rx_drops;
	4548	+ u64 tx_drops;
3539	4549
3540	4550	desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
3541	4551	adapter->last_keep_alive_jiffies = jiffies;
3542	4552
3543	4553	rx_drops = ((u64)desc->rx_drops_high << 32) \| desc->rx_drops_low;
	4554	+ tx_drops = ((u64)desc->tx_drops_high << 32) \| desc->tx_drops_low;
3544	4555
3545	4556	u64_stats_update_begin(&adapter->syncp);
3546	4557	/* These stats are accumulated by the device, so the counters indicate
3547	4558	* all drops since last reset.
3548	4559	*/
3549	4560	adapter->dev_stats.rx_drops = rx_drops;
	4561	+ adapter->dev_stats.tx_drops = tx_drops;
3550	4562	u64_stats_update_end(&adapter->syncp);
3551	4563	}
3552	4564
..	..	@@ -3561,7 +4573,7 @@
3561	4573	aenq_e->aenq_common_desc.group,
3562	4574	ENA_ADMIN_NOTIFICATION);
3563	4575
3564		- switch (aenq_e->aenq_common_desc.syndrom) {
	4576	+ switch (aenq_e->aenq_common_desc.syndrome) {
3565	4577	case ENA_ADMIN_UPDATE_HINTS:
3566	4578	hints = (struct ena_admin_ena_hw_hints *)
3567	4579	(&aenq_e->inline_data_w4);
..	..	@@ -3570,7 +4582,7 @@
3570	4582	default:
3571	4583	netif_err(adapter, drv, adapter->netdev,
3572	4584	"Invalid aenq notification link state %d\n",
3573		- aenq_e->aenq_common_desc.syndrom);
	4585	+ aenq_e->aenq_common_desc.syndrome);
3574	4586	}
3575	4587	}
3576	4588