~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* drivers/net/veth.c
3	4	*
..	..	@@ -24,6 +25,7 @@
24	25	#include <linux/filter.h>
25	26	#include <linux/ptr_ring.h>
26	27	#include <linux/bpf_trace.h>
	28	+#include <linux/net_tstamp.h>
27	29
28	30	#define DRV_NAME "veth"
29	31	#define DRV_VERSION "1.0"
..	..	@@ -32,13 +34,23 @@
32	34	#define VETH_RING_SIZE 256
33	35	#define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN)
34	36
35		-/* Separating two types of XDP xmit */
36		-#define VETH_XDP_TX BIT(0)
37		-#define VETH_XDP_REDIR BIT(1)
	37	+#define VETH_XDP_TX_BULK_SIZE 16
38	38
39		-struct pcpu_vstats {
40		- u64 packets;
41		- u64 bytes;
	39	+struct veth_stats {
	40	+ u64 rx_drops;
	41	+ /* xdp */
	42	+ u64 xdp_packets;
	43	+ u64 xdp_bytes;
	44	+ u64 xdp_redirect;
	45	+ u64 xdp_drops;
	46	+ u64 xdp_tx;
	47	+ u64 xdp_tx_err;
	48	+ u64 peer_tq_xdp_xmit;
	49	+ u64 peer_tq_xdp_xmit_err;
	50	+};
	51	+
	52	+struct veth_rq_stats {
	53	+ struct veth_stats vs;
42	54	struct u64_stats_sync syncp;
43	55	};
44	56
..	..	@@ -47,6 +59,7 @@
47	59	struct net_device *dev;
48	60	struct bpf_prog __rcu *xdp_prog;
49	61	struct xdp_mem_info xdp_mem;
	62	+ struct veth_rq_stats stats;
50	63	bool rx_notify_masked;
51	64	struct ptr_ring xdp_ring;
52	65	struct xdp_rxq_info xdp_rxq;
..	..	@@ -60,9 +73,40 @@
60	73	unsigned int requested_headroom;
61	74	};
62	75
	76	+struct veth_xdp_tx_bq {
	77	+ struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE];
	78	+ unsigned int count;
	79	+};
	80	+
63	81	/*
64	82	* ethtool interface
65	83	*/
	84	+
	85	+struct veth_q_stat_desc {
	86	+ char desc[ETH_GSTRING_LEN];
	87	+ size_t offset;
	88	+};
	89	+
	90	+#define VETH_RQ_STAT(m) offsetof(struct veth_stats, m)
	91	+
	92	+static const struct veth_q_stat_desc veth_rq_stats_desc[] = {
	93	+ { "xdp_packets", VETH_RQ_STAT(xdp_packets) },
	94	+ { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) },
	95	+ { "drops", VETH_RQ_STAT(rx_drops) },
	96	+ { "xdp_redirect", VETH_RQ_STAT(xdp_redirect) },
	97	+ { "xdp_drops", VETH_RQ_STAT(xdp_drops) },
	98	+ { "xdp_tx", VETH_RQ_STAT(xdp_tx) },
	99	+ { "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) },
	100	+};
	101	+
	102	+#define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc)
	103	+
	104	+static const struct veth_q_stat_desc veth_tq_stats_desc[] = {
	105	+ { "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) },
	106	+ { "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) },
	107	+};
	108	+
	109	+#define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc)
66	110
67	111	static struct {
68	112	const char string[ETH_GSTRING_LEN];
..	..	@@ -88,9 +132,29 @@
88	132
89	133	static void veth_get_strings(struct net_device dev, u32 stringset, u8 buf)
90	134	{
	135	+ char p = (char )buf;
	136	+ int i, j;
	137	+
91	138	switch(stringset) {
92	139	case ETH_SS_STATS:
93		- memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
	140	+ memcpy(p, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
	141	+ p += sizeof(ethtool_stats_keys);
	142	+ for (i = 0; i < dev->real_num_rx_queues; i++) {
	143	+ for (j = 0; j < VETH_RQ_STATS_LEN; j++) {
	144	+ snprintf(p, ETH_GSTRING_LEN,
	145	+ "rx_queue_%u_%.18s",
	146	+ i, veth_rq_stats_desc[j].desc);
	147	+ p += ETH_GSTRING_LEN;
	148	+ }
	149	+ }
	150	+ for (i = 0; i < dev->real_num_tx_queues; i++) {
	151	+ for (j = 0; j < VETH_TQ_STATS_LEN; j++) {
	152	+ snprintf(p, ETH_GSTRING_LEN,
	153	+ "tx_queue_%u_%.18s",
	154	+ i, veth_tq_stats_desc[j].desc);
	155	+ p += ETH_GSTRING_LEN;
	156	+ }
	157	+ }
94	158	break;
95	159	}
96	160	}
..	..	@@ -99,7 +163,9 @@
99	163	{
100	164	switch (sset) {
101	165	case ETH_SS_STATS:
102		- return ARRAY_SIZE(ethtool_stats_keys);
	166	+ return ARRAY_SIZE(ethtool_stats_keys) +
	167	+ VETH_RQ_STATS_LEN * dev->real_num_rx_queues +
	168	+ VETH_TQ_STATS_LEN * dev->real_num_tx_queues;
103	169	default:
104	170	return -EOPNOTSUPP;
105	171	}
..	..	@@ -108,10 +174,47 @@
108	174	static void veth_get_ethtool_stats(struct net_device *dev,
109	175	struct ethtool_stats stats, u64 data)
110	176	{
111		- struct veth_priv *priv = netdev_priv(dev);
	177	+ struct veth_priv rcv_priv, priv = netdev_priv(dev);
112	178	struct net_device *peer = rtnl_dereference(priv->peer);
	179	+ int i, j, idx;
113	180
114	181	data[0] = peer ? peer->ifindex : 0;
	182	+ idx = 1;
	183	+ for (i = 0; i < dev->real_num_rx_queues; i++) {
	184	+ const struct veth_rq_stats *rq_stats = &priv->rq[i].stats;
	185	+ const void stats_base = (void )&rq_stats->vs;
	186	+ unsigned int start;
	187	+ size_t offset;
	188	+
	189	+ do {
	190	+ start = u64_stats_fetch_begin_irq(&rq_stats->syncp);
	191	+ for (j = 0; j < VETH_RQ_STATS_LEN; j++) {
	192	+ offset = veth_rq_stats_desc[j].offset;
	193	+ data[idx + j] = (u64 )(stats_base + offset);
	194	+ }
	195	+ } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start));
	196	+ idx += VETH_RQ_STATS_LEN;
	197	+ }
	198	+
	199	+ if (!peer)
	200	+ return;
	201	+
	202	+ rcv_priv = netdev_priv(peer);
	203	+ for (i = 0; i < peer->real_num_rx_queues; i++) {
	204	+ const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats;
	205	+ const void base = (void )&rq_stats->vs;
	206	+ unsigned int start, tx_idx = idx;
	207	+ size_t offset;
	208	+
	209	+ tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN;
	210	+ do {
	211	+ start = u64_stats_fetch_begin_irq(&rq_stats->syncp);
	212	+ for (j = 0; j < VETH_TQ_STATS_LEN; j++) {
	213	+ offset = veth_tq_stats_desc[j].offset;
	214	+ data[tx_idx + j] += (u64 )(base + offset);
	215	+ }
	216	+ } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start));
	217	+ }
115	218	}
116	219
117	220	static const struct ethtool_ops veth_ethtool_ops = {
..	..	@@ -121,6 +224,7 @@
121	224	.get_sset_count = veth_get_sset_count,
122	225	.get_ethtool_stats = veth_get_ethtool_stats,
123	226	.get_link_ksettings = veth_get_link_ksettings,
	227	+ .get_ts_info = ethtool_op_get_ts_info,
124	228	};
125	229
126	230	/* general routines */
..	..	@@ -130,14 +234,14 @@
130	234	return (unsigned long)ptr & VETH_XDP_FLAG;
131	235	}
132	236
133		-static void veth_ptr_to_xdp(void ptr)
	237	+static struct xdp_frame veth_ptr_to_xdp(void ptr)
134	238	{
135	239	return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG);
136	240	}
137	241
138		-static void veth_xdp_to_ptr(void ptr)
	242	+static void veth_xdp_to_ptr(struct xdp_frame xdp)
139	243	{
140		- return (void *)((unsigned long)ptr \| VETH_XDP_FLAG);
	244	+ return (void *)((unsigned long)xdp \| VETH_XDP_FLAG);
141	245	}
142	246
143	247	static void veth_ptr_free(void *ptr)
..	..	@@ -188,7 +292,7 @@
188	292
189	293	rcu_read_lock();
190	294	rcv = rcu_dereference(priv->peer);
191		- if (unlikely(!rcv)) {
	295	+ if (unlikely(!rcv) \|\| !pskb_may_pull(skb, ETH_HLEN)) {
192	296	kfree_skb(skb);
193	297	goto drop;
194	298	}
..	..	@@ -200,13 +304,10 @@
200	304	rcv_xdp = rcu_access_pointer(rq->xdp_prog);
201	305	}
202	306
	307	+ skb_tx_timestamp(skb);
203	308	if (likely(veth_forward_skb(rcv, skb, rq, rcv_xdp) == NET_RX_SUCCESS)) {
204		- struct pcpu_vstats *stats = this_cpu_ptr(dev->vstats);
205		-
206		- u64_stats_update_begin(&stats->syncp);
207		- stats->bytes += length;
208		- stats->packets++;
209		- u64_stats_update_end(&stats->syncp);
	309	+ if (!rcv_xdp)
	310	+ dev_lstats_add(dev, length);
210	311	} else {
211	312	drop:
212	313	atomic64_inc(&priv->dropped);
..	..	@@ -220,27 +321,43 @@
220	321	return NETDEV_TX_OK;
221	322	}
222	323
223		-static u64 veth_stats_one(struct pcpu_vstats result, struct net_device dev)
	324	+static u64 veth_stats_tx(struct net_device dev, u64 packets, u64 *bytes)
224	325	{
225	326	struct veth_priv *priv = netdev_priv(dev);
226		- int cpu;
227	327
228		- result->packets = 0;
229		- result->bytes = 0;
230		- for_each_possible_cpu(cpu) {
231		- struct pcpu_vstats *stats = per_cpu_ptr(dev->vstats, cpu);
232		- u64 packets, bytes;
	328	+ dev_lstats_read(dev, packets, bytes);
	329	+ return atomic64_read(&priv->dropped);
	330	+}
	331	+
	332	+static void veth_stats_rx(struct veth_stats result, struct net_device dev)
	333	+{
	334	+ struct veth_priv *priv = netdev_priv(dev);
	335	+ int i;
	336	+
	337	+ result->peer_tq_xdp_xmit_err = 0;
	338	+ result->xdp_packets = 0;
	339	+ result->xdp_tx_err = 0;
	340	+ result->xdp_bytes = 0;
	341	+ result->rx_drops = 0;
	342	+ for (i = 0; i < dev->num_rx_queues; i++) {
	343	+ u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err;
	344	+ struct veth_rq_stats *stats = &priv->rq[i].stats;
233	345	unsigned int start;
234	346
235	347	do {
236	348	start = u64_stats_fetch_begin_irq(&stats->syncp);
237		- packets = stats->packets;
238		- bytes = stats->bytes;
	349	+ peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err;
	350	+ xdp_tx_err = stats->vs.xdp_tx_err;
	351	+ packets = stats->vs.xdp_packets;
	352	+ bytes = stats->vs.xdp_bytes;
	353	+ drops = stats->vs.rx_drops;
239	354	} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
240		- result->packets += packets;
241		- result->bytes += bytes;
	355	+ result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err;
	356	+ result->xdp_tx_err += xdp_tx_err;
	357	+ result->xdp_packets += packets;
	358	+ result->xdp_bytes += bytes;
	359	+ result->rx_drops += drops;
242	360	}
243		- return atomic64_read(&priv->dropped);
244	361	}
245	362
246	363	static void veth_get_stats64(struct net_device *dev,
..	..	@@ -248,18 +365,31 @@
248	365	{
249	366	struct veth_priv *priv = netdev_priv(dev);
250	367	struct net_device *peer;
251		- struct pcpu_vstats one;
	368	+ struct veth_stats rx;
	369	+ u64 packets, bytes;
252	370
253		- tot->tx_dropped = veth_stats_one(&one, dev);
254		- tot->tx_bytes = one.bytes;
255		- tot->tx_packets = one.packets;
	371	+ tot->tx_dropped = veth_stats_tx(dev, &packets, &bytes);
	372	+ tot->tx_bytes = bytes;
	373	+ tot->tx_packets = packets;
	374	+
	375	+ veth_stats_rx(&rx, dev);
	376	+ tot->tx_dropped += rx.xdp_tx_err;
	377	+ tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err;
	378	+ tot->rx_bytes = rx.xdp_bytes;
	379	+ tot->rx_packets = rx.xdp_packets;
256	380
257	381	rcu_read_lock();
258	382	peer = rcu_dereference(priv->peer);
259	383	if (peer) {
260		- tot->rx_dropped = veth_stats_one(&one, peer);
261		- tot->rx_bytes = one.bytes;
262		- tot->rx_packets = one.packets;
	384	+ veth_stats_tx(peer, &packets, &bytes);
	385	+ tot->rx_bytes += bytes;
	386	+ tot->rx_packets += packets;
	387	+
	388	+ veth_stats_rx(&rx, peer);
	389	+ tot->tx_dropped += rx.peer_tq_xdp_xmit_err;
	390	+ tot->rx_dropped += rx.xdp_tx_err;
	391	+ tot->tx_bytes += rx.xdp_bytes;
	392	+ tot->tx_packets += rx.xdp_packets;
263	393	}
264	394	rcu_read_unlock();
265	395	}
..	..	@@ -274,10 +404,6 @@
274	404	{
275	405	struct sk_buff *skb;
276	406
277		- if (!buflen) {
278		- buflen = SKB_DATA_ALIGN(headroom + len) +
279		- SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
280		- }
281	407	skb = build_skb(head, buflen);
282	408	if (!skb)
283	409	return NULL;
..	..	@@ -293,21 +419,31 @@
293	419	return smp_processor_id() % dev->real_num_rx_queues;
294	420	}
295	421
	422	+static struct net_device veth_peer_dev(struct net_device dev)
	423	+{
	424	+ struct veth_priv *priv = netdev_priv(dev);
	425	+
	426	+ /* Callers must be under RCU read side. */
	427	+ return rcu_dereference(priv->peer);
	428	+}
	429	+
296	430	static int veth_xdp_xmit(struct net_device *dev, int n,
297		- struct xdp_frame **frames, u32 flags)
	431	+ struct xdp_frame **frames,
	432	+ u32 flags, bool ndo_xmit)
298	433	{
299	434	struct veth_priv rcv_priv, priv = netdev_priv(dev);
	435	+ int i, ret = -ENXIO, drops = 0;
300	436	struct net_device *rcv;
301	437	unsigned int max_len;
302	438	struct veth_rq *rq;
303		- int i, drops = 0;
304	439
305	440	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
306	441	return -EINVAL;
307	442
	443	+ rcu_read_lock();
308	444	rcv = rcu_dereference(priv->peer);
309	445	if (unlikely(!rcv))
310		- return -ENXIO;
	446	+ goto out;
311	447
312	448	rcv_priv = netdev_priv(rcv);
313	449	rq = &rcv_priv->rq[veth_select_rxq(rcv)];
..	..	@@ -316,7 +452,7 @@
316	452	* device is up.
317	453	*/
318	454	if (!rcu_access_pointer(rq->xdp_prog))
319		- return -ENXIO;
	455	+ goto out;
320	456
321	457	max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN;
322	458
..	..	@@ -336,44 +472,99 @@
336	472	if (flags & XDP_XMIT_FLUSH)
337	473	__veth_xdp_flush(rq);
338	474
339		- return n - drops;
	475	+ ret = n - drops;
	476	+ if (ndo_xmit) {
	477	+ u64_stats_update_begin(&rq->stats.syncp);
	478	+ rq->stats.vs.peer_tq_xdp_xmit += n - drops;
	479	+ rq->stats.vs.peer_tq_xdp_xmit_err += drops;
	480	+ u64_stats_update_end(&rq->stats.syncp);
	481	+ }
	482	+
	483	+out:
	484	+ rcu_read_unlock();
	485	+
	486	+ return ret;
340	487	}
341	488
342		-static void veth_xdp_flush(struct net_device *dev)
	489	+static int veth_ndo_xdp_xmit(struct net_device *dev, int n,
	490	+ struct xdp_frame **frames, u32 flags)
343	491	{
344		- struct veth_priv rcv_priv, priv = netdev_priv(dev);
	492	+ int err;
	493	+
	494	+ err = veth_xdp_xmit(dev, n, frames, flags, true);
	495	+ if (err < 0) {
	496	+ struct veth_priv *priv = netdev_priv(dev);
	497	+
	498	+ atomic64_add(n, &priv->dropped);
	499	+ }
	500	+
	501	+ return err;
	502	+}
	503	+
	504	+static void veth_xdp_flush_bq(struct veth_rq rq, struct veth_xdp_tx_bq bq)
	505	+{
	506	+ int sent, i, err = 0;
	507	+
	508	+ sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false);
	509	+ if (sent < 0) {
	510	+ err = sent;
	511	+ sent = 0;
	512	+ for (i = 0; i < bq->count; i++)
	513	+ xdp_return_frame(bq->q[i]);
	514	+ }
	515	+ trace_xdp_bulk_tx(rq->dev, sent, bq->count - sent, err);
	516	+
	517	+ u64_stats_update_begin(&rq->stats.syncp);
	518	+ rq->stats.vs.xdp_tx += sent;
	519	+ rq->stats.vs.xdp_tx_err += bq->count - sent;
	520	+ u64_stats_update_end(&rq->stats.syncp);
	521	+
	522	+ bq->count = 0;
	523	+}
	524	+
	525	+static void veth_xdp_flush(struct veth_rq rq, struct veth_xdp_tx_bq bq)
	526	+{
	527	+ struct veth_priv rcv_priv, priv = netdev_priv(rq->dev);
345	528	struct net_device *rcv;
346		- struct veth_rq *rq;
	529	+ struct veth_rq *rcv_rq;
347	530
348	531	rcu_read_lock();
	532	+ veth_xdp_flush_bq(rq, bq);
349	533	rcv = rcu_dereference(priv->peer);
350	534	if (unlikely(!rcv))
351	535	goto out;
352	536
353	537	rcv_priv = netdev_priv(rcv);
354		- rq = &rcv_priv->rq[veth_select_rxq(rcv)];
	538	+ rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)];
355	539	/* xdp_ring is initialized on receive side? */
356		- if (unlikely(!rcu_access_pointer(rq->xdp_prog)))
	540	+ if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog)))
357	541	goto out;
358	542
359		- __veth_xdp_flush(rq);
	543	+ __veth_xdp_flush(rcv_rq);
360	544	out:
361	545	rcu_read_unlock();
362	546	}
363	547
364		-static int veth_xdp_tx(struct net_device dev, struct xdp_buff xdp)
	548	+static int veth_xdp_tx(struct veth_rq rq, struct xdp_buff xdp,
	549	+ struct veth_xdp_tx_bq *bq)
365	550	{
366		- struct xdp_frame *frame = convert_to_xdp_frame(xdp);
	551	+ struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp);
367	552
368	553	if (unlikely(!frame))
369	554	return -EOVERFLOW;
370	555
371		- return veth_xdp_xmit(dev, 1, &frame, 0);
	556	+ if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE))
	557	+ veth_xdp_flush_bq(rq, bq);
	558	+
	559	+ bq->q[bq->count++] = frame;
	560	+
	561	+ return 0;
372	562	}
373	563
374	564	static struct sk_buff veth_xdp_rcv_one(struct veth_rq rq,
375	565	struct xdp_frame *frame,
376		- unsigned int *xdp_xmit)
	566	+ struct veth_xdp_tx_bq *bq,
	567	+ struct veth_stats *stats)
377	568	{
378	569	void *hard_start = frame->data - frame->headroom;
379	570	int len = frame->len, delta = 0;
..	..	@@ -391,10 +582,7 @@
391	582	struct xdp_buff xdp;
392	583	u32 act;
393	584
394		- xdp.data_hard_start = hard_start;
395		- xdp.data = frame->data;
396		- xdp.data_end = frame->data + frame->len;
397		- xdp.data_meta = frame->data - frame->metasize;
	585	+ xdp_convert_frame_to_buff(frame, &xdp);
398	586	xdp.rxq = &rq->xdp_rxq;
399	587
400	588	act = bpf_prog_run_xdp(xdp_prog, &xdp);
..	..	@@ -407,12 +595,13 @@
407	595	case XDP_TX:
408	596	orig_frame = *frame;
409	597	xdp.rxq->mem = frame->mem;
410		- if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) {
	598	+ if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) {
411	599	trace_xdp_exception(rq->dev, xdp_prog, act);
412	600	frame = &orig_frame;
	601	+ stats->rx_drops++;
413	602	goto err_xdp;
414	603	}
415		- *xdp_xmit \|= VETH_XDP_TX;
	604	+ stats->xdp_tx++;
416	605	rcu_read_unlock();
417	606	goto xdp_xmit;
418	607	case XDP_REDIRECT:
..	..	@@ -420,28 +609,34 @@
420	609	xdp.rxq->mem = frame->mem;
421	610	if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) {
422	611	frame = &orig_frame;
	612	+ stats->rx_drops++;
423	613	goto err_xdp;
424	614	}
425		- *xdp_xmit \|= VETH_XDP_REDIR;
	615	+ stats->xdp_redirect++;
426	616	rcu_read_unlock();
427	617	goto xdp_xmit;
428	618	default:
429	619	bpf_warn_invalid_xdp_action(act);
	620	+ fallthrough;
430	621	case XDP_ABORTED:
431	622	trace_xdp_exception(rq->dev, xdp_prog, act);
	623	+ fallthrough;
432	624	case XDP_DROP:
	625	+ stats->xdp_drops++;
433	626	goto err_xdp;
434	627	}
435	628	}
436	629	rcu_read_unlock();
437	630
438	631	headroom = sizeof(struct xdp_frame) + frame->headroom - delta;
439		- skb = veth_build_skb(hard_start, headroom, len, 0);
	632	+ skb = veth_build_skb(hard_start, headroom, len, frame->frame_sz);
440	633	if (!skb) {
441	634	xdp_return_frame(frame);
	635	+ stats->rx_drops++;
442	636	goto err;
443	637	}
444	638
	639	+ xdp_release_frame(frame);
445	640	xdp_scrub_frame(frame);
446	641	skb->protocol = eth_type_trans(skb, rq->dev);
447	642	err:
..	..	@@ -453,8 +648,10 @@
453	648	return NULL;
454	649	}
455	650
456		-static struct sk_buff veth_xdp_rcv_skb(struct veth_rq rq, struct sk_buff *skb,
457		- unsigned int *xdp_xmit)
	651	+static struct sk_buff veth_xdp_rcv_skb(struct veth_rq rq,
	652	+ struct sk_buff *skb,
	653	+ struct veth_xdp_tx_bq *bq,
	654	+ struct veth_stats *stats)
458	655	{
459	656	u32 pktlen, headroom, act, metalen;
460	657	void orig_data, orig_data_end;
..	..	@@ -498,9 +695,8 @@
498	695	goto drop;
499	696	}
500	697
501		- nskb = veth_build_skb(head,
502		- VETH_XDP_HEADROOM + mac_len, skb->len,
503		- PAGE_SIZE);
	698	+ nskb = veth_build_skb(head, VETH_XDP_HEADROOM + mac_len,
	699	+ skb->len, PAGE_SIZE);
504	700	if (!nskb) {
505	701	page_frag_free(head);
506	702	goto drop;
..	..	@@ -518,6 +714,11 @@
518	714	xdp.data_end = xdp.data + pktlen;
519	715	xdp.data_meta = xdp.data;
520	716	xdp.rxq = &rq->xdp_rxq;
	717	+
	718	+ /* SKB "head" area always have tailroom for skb_shared_info */
	719	+ xdp.frame_sz = (void *)skb_end_pointer(skb) - xdp.data_hard_start;
	720	+ xdp.frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	721	+
521	722	orig_data = xdp.data;
522	723	orig_data_end = xdp.data_end;
523	724
..	..	@@ -530,31 +731,38 @@
530	731	get_page(virt_to_page(xdp.data));
531	732	consume_skb(skb);
532	733	xdp.rxq->mem = rq->xdp_mem;
533		- if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) {
	734	+ if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) {
534	735	trace_xdp_exception(rq->dev, xdp_prog, act);
	736	+ stats->rx_drops++;
535	737	goto err_xdp;
536	738	}
537		- *xdp_xmit \|= VETH_XDP_TX;
	739	+ stats->xdp_tx++;
538	740	rcu_read_unlock();
539	741	goto xdp_xmit;
540	742	case XDP_REDIRECT:
541	743	get_page(virt_to_page(xdp.data));
542	744	consume_skb(skb);
543	745	xdp.rxq->mem = rq->xdp_mem;
544		- if (xdp_do_redirect(rq->dev, &xdp, xdp_prog))
	746	+ if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) {
	747	+ stats->rx_drops++;
545	748	goto err_xdp;
546		- *xdp_xmit \|= VETH_XDP_REDIR;
	749	+ }
	750	+ stats->xdp_redirect++;
547	751	rcu_read_unlock();
548	752	goto xdp_xmit;
549	753	default:
550	754	bpf_warn_invalid_xdp_action(act);
	755	+ fallthrough;
551	756	case XDP_ABORTED:
552	757	trace_xdp_exception(rq->dev, xdp_prog, act);
	758	+ fallthrough;
553	759	case XDP_DROP:
554		- goto drop;
	760	+ stats->xdp_drops++;
	761	+ goto xdp_drop;
555	762	}
556	763	rcu_read_unlock();
557	764
	765	+ /* check if bpf_xdp_adjust_head was used */
558	766	delta = orig_data - xdp.data;
559	767	off = mac_len + delta;
560	768	if (off > 0)
..	..	@@ -562,9 +770,11 @@
562	770	else if (off < 0)
563	771	__skb_pull(skb, -off);
564	772	skb->mac_header -= delta;
	773	+
	774	+ /* check if bpf_xdp_adjust_tail was used */
565	775	off = xdp.data_end - orig_data_end;
566	776	if (off != 0)
567		- __skb_put(skb, off);
	777	+ __skb_put(skb, off); /* positive on grow, negative on shrink */
568	778	skb->protocol = eth_type_trans(skb, rq->dev);
569	779
570	780	metalen = xdp.data - xdp.data_meta;
..	..	@@ -573,6 +783,8 @@
573	783	out:
574	784	return skb;
575	785	drop:
	786	+ stats->rx_drops++;
	787	+xdp_drop:
576	788	rcu_read_unlock();
577	789	kfree_skb(skb);
578	790	return NULL;
..	..	@@ -583,7 +795,9 @@
583	795	return NULL;
584	796	}
585	797
586		-static int veth_xdp_rcv(struct veth_rq rq, int budget, unsigned int xdp_xmit)
	798	+static int veth_xdp_rcv(struct veth_rq *rq, int budget,
	799	+ struct veth_xdp_tx_bq *bq,
	800	+ struct veth_stats *stats)
587	801	{
588	802	int i, done = 0;
589	803
..	..	@@ -595,10 +809,14 @@
595	809	break;
596	810
597	811	if (veth_is_xdp_frame(ptr)) {
598		- skb = veth_xdp_rcv_one(rq, veth_ptr_to_xdp(ptr),
599		- xdp_xmit);
	812	+ struct xdp_frame *frame = veth_ptr_to_xdp(ptr);
	813	+
	814	+ stats->xdp_bytes += frame->len;
	815	+ skb = veth_xdp_rcv_one(rq, frame, bq, stats);
600	816	} else {
601		- skb = veth_xdp_rcv_skb(rq, ptr, xdp_xmit);
	817	+ skb = ptr;
	818	+ stats->xdp_bytes += skb->len;
	819	+ skb = veth_xdp_rcv_skb(rq, skb, bq, stats);
602	820	}
603	821
604	822	if (skb)
..	..	@@ -607,6 +825,14 @@
607	825	done++;
608	826	}
609	827
	828	+ u64_stats_update_begin(&rq->stats.syncp);
	829	+ rq->stats.vs.xdp_redirect += stats->xdp_redirect;
	830	+ rq->stats.vs.xdp_bytes += stats->xdp_bytes;
	831	+ rq->stats.vs.xdp_drops += stats->xdp_drops;
	832	+ rq->stats.vs.rx_drops += stats->rx_drops;
	833	+ rq->stats.vs.xdp_packets += done;
	834	+ u64_stats_update_end(&rq->stats.syncp);
	835	+
610	836	return done;
611	837	}
612	838
..	..	@@ -614,11 +840,14 @@
614	840	{
615	841	struct veth_rq *rq =
616	842	container_of(napi, struct veth_rq, xdp_napi);
617		- unsigned int xdp_xmit = 0;
	843	+ struct veth_stats stats = {};
	844	+ struct veth_xdp_tx_bq bq;
618	845	int done;
619	846
	847	+ bq.count = 0;
	848	+
620	849	xdp_set_return_frame_no_direct();
621		- done = veth_xdp_rcv(rq, budget, &xdp_xmit);
	850	+ done = veth_xdp_rcv(rq, budget, &bq, &stats);
622	851
623	852	if (done < budget && napi_complete_done(napi, done)) {
624	853	/* Write rx_notify_masked before reading ptr_ring */
..	..	@@ -631,10 +860,10 @@
631	860	}
632	861	}
633	862
634		- if (xdp_xmit & VETH_XDP_TX)
635		- veth_xdp_flush(rq->dev);
636		- if (xdp_xmit & VETH_XDP_REDIR)
637		- xdp_do_flush_map();
	863	+ if (stats.xdp_tx > 0)
	864	+ veth_xdp_flush(rq, &bq);
	865	+ if (stats.xdp_redirect > 0)
	866	+ xdp_do_flush();
638	867	xdp_clear_return_frame_no_direct();
639	868
640	869	return done;
..	..	@@ -677,14 +906,13 @@
677	906	struct veth_rq *rq = &priv->rq[i];
678	907
679	908	napi_disable(&rq->xdp_napi);
680		- napi_hash_del(&rq->xdp_napi);
	909	+ __netif_napi_del(&rq->xdp_napi);
681	910	}
682	911	synchronize_net();
683	912
684	913	for (i = 0; i < dev->real_num_rx_queues; i++) {
685	914	struct veth_rq *rq = &priv->rq[i];
686	915
687		- netif_napi_del(&rq->xdp_napi);
688	916	rq->rx_notify_masked = false;
689	917	ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free);
690	918	}
..	..	@@ -799,8 +1027,10 @@
799	1027	if (!priv->rq)
800	1028	return -ENOMEM;
801	1029
802		- for (i = 0; i < dev->num_rx_queues; i++)
	1030	+ for (i = 0; i < dev->num_rx_queues; i++) {
803	1031	priv->rq[i].dev = dev;
	1032	+ u64_stats_init(&priv->rq[i].stats.syncp);
	1033	+ }
804	1034
805	1035	return 0;
806	1036	}
..	..	@@ -816,13 +1046,13 @@
816	1046	{
817	1047	int err;
818	1048
819		- dev->vstats = netdev_alloc_pcpu_stats(struct pcpu_vstats);
820		- if (!dev->vstats)
	1049	+ dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
	1050	+ if (!dev->lstats)
821	1051	return -ENOMEM;
822	1052
823	1053	err = veth_alloc_queues(dev);
824	1054	if (err) {
825		- free_percpu(dev->vstats);
	1055	+ free_percpu(dev->lstats);
826	1056	return err;
827	1057	}
828	1058
..	..	@@ -832,7 +1062,7 @@
832	1062	static void veth_dev_free(struct net_device *dev)
833	1063	{
834	1064	veth_free_queues(dev);
835		- free_percpu(dev->vstats);
	1065	+ free_percpu(dev->lstats);
836	1066	}
837	1067
838	1068	#ifdef CONFIG_NET_POLL_CONTROLLER
..	..	@@ -976,26 +1206,11 @@
976	1206	return err;
977	1207	}
978	1208
979		-static u32 veth_xdp_query(struct net_device *dev)
980		-{
981		- struct veth_priv *priv = netdev_priv(dev);
982		- const struct bpf_prog *xdp_prog;
983		-
984		- xdp_prog = priv->_xdp_prog;
985		- if (xdp_prog)
986		- return xdp_prog->aux->id;
987		-
988		- return 0;
989		-}
990		-
991	1209	static int veth_xdp(struct net_device dev, struct netdev_bpf xdp)
992	1210	{
993	1211	switch (xdp->command) {
994	1212	case XDP_SETUP_PROG:
995	1213	return veth_xdp_set(dev, xdp->prog, xdp->extack);
996		- case XDP_QUERY_PROG:
997		- xdp->prog_id = veth_xdp_query(dev);
998		- return 0;
999	1214	default:
1000	1215	return -EINVAL;
1001	1216	}
..	..	@@ -1017,7 +1232,8 @@
1017	1232	.ndo_features_check = passthru_features_check,
1018	1233	.ndo_set_rx_headroom = veth_set_rx_headroom,
1019	1234	.ndo_bpf = veth_xdp,
1020		- .ndo_xdp_xmit = veth_xdp_xmit,
	1235	+ .ndo_xdp_xmit = veth_ndo_xdp_xmit,
	1236	+ .ndo_get_peer_dev = veth_peer_dev,
1021	1237	};
1022	1238
1023	1239	#define VETH_FEATURES (NETIF_F_SG \| NETIF_F_FRAGLIST \| NETIF_F_HW_CSUM \| \
..	..	@@ -1126,7 +1342,7 @@
1126	1342	return PTR_ERR(net);
1127	1343
1128	1344	peer = rtnl_create_link(net, ifname, name_assign_type,
1129		- &veth_link_ops, tbp);
	1345	+ &veth_link_ops, tbp, extack);
1130	1346	if (IS_ERR(peer)) {
1131	1347	put_net(net);
1132	1348	return PTR_ERR(peer);