~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* drivers/net/veth.c
3	4	*
..	..	@@ -24,6 +25,7 @@
24	25	#include <linux/filter.h>
25	26	#include <linux/ptr_ring.h>
26	27	#include <linux/bpf_trace.h>
	28	+#include <linux/net_tstamp.h>
27	29
28	30	#define DRV_NAME "veth"
29	31	#define DRV_VERSION "1.0"
..	..	@@ -32,13 +34,23 @@
32	34	#define VETH_RING_SIZE 256
33	35	#define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN)
34	36
35		-/* Separating two types of XDP xmit */
36		-#define VETH_XDP_TX BIT(0)
37		-#define VETH_XDP_REDIR BIT(1)
	37	+#define VETH_XDP_TX_BULK_SIZE 16
38	38
39		-struct pcpu_vstats {
40		- u64 packets;
41		- u64 bytes;
	39	+struct veth_stats {
	40	+ u64 rx_drops;
	41	+ /* xdp */
	42	+ u64 xdp_packets;
	43	+ u64 xdp_bytes;
	44	+ u64 xdp_redirect;
	45	+ u64 xdp_drops;
	46	+ u64 xdp_tx;
	47	+ u64 xdp_tx_err;
	48	+ u64 peer_tq_xdp_xmit;
	49	+ u64 peer_tq_xdp_xmit_err;
	50	+};
	51	+
	52	+struct veth_rq_stats {
	53	+ struct veth_stats vs;
42	54	struct u64_stats_sync syncp;
43	55	};
44	56
..	..	@@ -47,6 +59,7 @@
47	59	struct net_device *dev;
48	60	struct bpf_prog __rcu *xdp_prog;
49	61	struct xdp_mem_info xdp_mem;
	62	+ struct veth_rq_stats stats;
50	63	bool rx_notify_masked;
51	64	struct ptr_ring xdp_ring;
52	65	struct xdp_rxq_info xdp_rxq;
..	..	@@ -60,9 +73,40 @@
60	73	unsigned int requested_headroom;
61	74	};
62	75
	76	+struct veth_xdp_tx_bq {
	77	+ struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE];
	78	+ unsigned int count;
	79	+};
	80	+
63	81	/*
64	82	* ethtool interface
65	83	*/
	84	+
	85	+struct veth_q_stat_desc {
	86	+ char desc[ETH_GSTRING_LEN];
	87	+ size_t offset;
	88	+};
	89	+
	90	+#define VETH_RQ_STAT(m) offsetof(struct veth_stats, m)
	91	+
	92	+static const struct veth_q_stat_desc veth_rq_stats_desc[] = {
	93	+ { "xdp_packets", VETH_RQ_STAT(xdp_packets) },
	94	+ { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) },
	95	+ { "drops", VETH_RQ_STAT(rx_drops) },
	96	+ { "xdp_redirect", VETH_RQ_STAT(xdp_redirect) },
	97	+ { "xdp_drops", VETH_RQ_STAT(xdp_drops) },
	98	+ { "xdp_tx", VETH_RQ_STAT(xdp_tx) },
	99	+ { "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) },
	100	+};
	101	+
	102	+#define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc)
	103	+
	104	+static const struct veth_q_stat_desc veth_tq_stats_desc[] = {
	105	+ { "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) },
	106	+ { "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) },
	107	+};
	108	+
	109	+#define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc)
66	110
67	111	static struct {
68	112	const char string[ETH_GSTRING_LEN];
..	..	@@ -88,9 +132,29 @@
88	132
89	133	static void veth_get_strings(struct net_device dev, u32 stringset, u8 buf)
90	134	{
	135	+ char p = (char )buf;
	136	+ int i, j;
	137	+
91	138	switch(stringset) {
92	139	case ETH_SS_STATS:
93		- memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
	140	+ memcpy(p, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
	141	+ p += sizeof(ethtool_stats_keys);
	142	+ for (i = 0; i < dev->real_num_rx_queues; i++) {
	143	+ for (j = 0; j < VETH_RQ_STATS_LEN; j++) {
	144	+ snprintf(p, ETH_GSTRING_LEN,
	145	+ "rx_queue_%u_%.18s",
	146	+ i, veth_rq_stats_desc[j].desc);
	147	+ p += ETH_GSTRING_LEN;
	148	+ }
	149	+ }
	150	+ for (i = 0; i < dev->real_num_tx_queues; i++) {
	151	+ for (j = 0; j < VETH_TQ_STATS_LEN; j++) {
	152	+ snprintf(p, ETH_GSTRING_LEN,
	153	+ "tx_queue_%u_%.18s",
	154	+ i, veth_tq_stats_desc[j].desc);
	155	+ p += ETH_GSTRING_LEN;
	156	+ }
	157	+ }
94	158	break;
95	159	}
96	160	}
..	..	@@ -99,7 +163,9 @@
99	163	{
100	164	switch (sset) {
101	165	case ETH_SS_STATS:
102		- return ARRAY_SIZE(ethtool_stats_keys);
	166	+ return ARRAY_SIZE(ethtool_stats_keys) +
	167	+ VETH_RQ_STATS_LEN * dev->real_num_rx_queues +
	168	+ VETH_TQ_STATS_LEN * dev->real_num_tx_queues;
103	169	default:
104	170	return -EOPNOTSUPP;
105	171	}
..	..	@@ -108,10 +174,47 @@
108	174	static void veth_get_ethtool_stats(struct net_device *dev,
109	175	struct ethtool_stats stats, u64 data)
110	176	{
111		- struct veth_priv *priv = netdev_priv(dev);
	177	+ struct veth_priv rcv_priv, priv = netdev_priv(dev);
112	178	struct net_device *peer = rtnl_dereference(priv->peer);
	179	+ int i, j, idx;
113	180
114	181	data[0] = peer ? peer->ifindex : 0;
	182	+ idx = 1;
	183	+ for (i = 0; i < dev->real_num_rx_queues; i++) {
	184	+ const struct veth_rq_stats *rq_stats = &priv->rq[i].stats;
	185	+ const void stats_base = (void )&rq_stats->vs;
	186	+ unsigned int start;
	187	+ size_t offset;
	188	+
	189	+ do {
	190	+ start = u64_stats_fetch_begin_irq(&rq_stats->syncp);
	191	+ for (j = 0; j < VETH_RQ_STATS_LEN; j++) {
	192	+ offset = veth_rq_stats_desc[j].offset;
	193	+ data[idx + j] = (u64 )(stats_base + offset);
	194	+ }
	195	+ } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start));
	196	+ idx += VETH_RQ_STATS_LEN;
	197	+ }
	198	+
	199	+ if (!peer)
	200	+ return;
	201	+
	202	+ rcv_priv = netdev_priv(peer);
	203	+ for (i = 0; i < peer->real_num_rx_queues; i++) {
	204	+ const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats;
	205	+ const void base = (void )&rq_stats->vs;
	206	+ unsigned int start, tx_idx = idx;
	207	+ size_t offset;
	208	+
	209	+ tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN;
	210	+ do {
	211	+ start = u64_stats_fetch_begin_irq(&rq_stats->syncp);
	212	+ for (j = 0; j < VETH_TQ_STATS_LEN; j++) {
	213	+ offset = veth_tq_stats_desc[j].offset;
	214	+ data[tx_idx + j] += (u64 )(base + offset);
	215	+ }
	216	+ } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start));
	217	+ }
115	218	}
116	219
117	220	static const struct ethtool_ops veth_ethtool_ops = {
..	..	@@ -121,6 +224,7 @@
121	224	.get_sset_count = veth_get_sset_count,
122	225	.get_ethtool_stats = veth_get_ethtool_stats,
123	226	.get_link_ksettings = veth_get_link_ksettings,
	227	+ .get_ts_info = ethtool_op_get_ts_info,
124	228	};
125	229
126	230	/* general routines */
..	..	@@ -130,14 +234,14 @@
130	234	return (unsigned long)ptr & VETH_XDP_FLAG;
131	235	}
132	236
133		-static void veth_ptr_to_xdp(void ptr)
	237	+static struct xdp_frame veth_ptr_to_xdp(void ptr)
134	238	{
135	239	return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG);
136	240	}
137	241
138		-static void veth_xdp_to_ptr(void ptr)
	242	+static void veth_xdp_to_ptr(struct xdp_frame xdp)
139	243	{
140		- return (void *)((unsigned long)ptr \| VETH_XDP_FLAG);
	244	+ return (void *)((unsigned long)xdp \| VETH_XDP_FLAG);
141	245	}
142	246
143	247	static void veth_ptr_free(void *ptr)
..	..	@@ -181,6 +285,7 @@
181	285	{
182	286	struct veth_priv rcv_priv, priv = netdev_priv(dev);
183	287	struct veth_rq *rq = NULL;
	288	+ int ret = NETDEV_TX_OK;
184	289	struct net_device *rcv;
185	290	int length = skb->len;
186	291	bool rcv_xdp = false;
..	..	@@ -188,7 +293,7 @@
188	293
189	294	rcu_read_lock();
190	295	rcv = rcu_dereference(priv->peer);
191		- if (unlikely(!rcv)) {
	296	+ if (unlikely(!rcv) \|\| !pskb_may_pull(skb, ETH_HLEN)) {
192	297	kfree_skb(skb);
193	298	goto drop;
194	299	}
..	..	@@ -200,16 +305,14 @@
200	305	rcv_xdp = rcu_access_pointer(rq->xdp_prog);
201	306	}
202	307
	308	+ skb_tx_timestamp(skb);
203	309	if (likely(veth_forward_skb(rcv, skb, rq, rcv_xdp) == NET_RX_SUCCESS)) {
204		- struct pcpu_vstats *stats = this_cpu_ptr(dev->vstats);
205		-
206		- u64_stats_update_begin(&stats->syncp);
207		- stats->bytes += length;
208		- stats->packets++;
209		- u64_stats_update_end(&stats->syncp);
	310	+ if (!rcv_xdp)
	311	+ dev_lstats_add(dev, length);
210	312	} else {
211	313	drop:
212	314	atomic64_inc(&priv->dropped);
	315	+ ret = NET_XMIT_DROP;
213	316	}
214	317
215	318	if (rcv_xdp)
..	..	@@ -217,30 +320,46 @@
217	320
218	321	rcu_read_unlock();
219	322
220		- return NETDEV_TX_OK;
	323	+ return ret;
221	324	}
222	325
223		-static u64 veth_stats_one(struct pcpu_vstats result, struct net_device dev)
	326	+static u64 veth_stats_tx(struct net_device dev, u64 packets, u64 *bytes)
224	327	{
225	328	struct veth_priv *priv = netdev_priv(dev);
226		- int cpu;
227	329
228		- result->packets = 0;
229		- result->bytes = 0;
230		- for_each_possible_cpu(cpu) {
231		- struct pcpu_vstats *stats = per_cpu_ptr(dev->vstats, cpu);
232		- u64 packets, bytes;
	330	+ dev_lstats_read(dev, packets, bytes);
	331	+ return atomic64_read(&priv->dropped);
	332	+}
	333	+
	334	+static void veth_stats_rx(struct veth_stats result, struct net_device dev)
	335	+{
	336	+ struct veth_priv *priv = netdev_priv(dev);
	337	+ int i;
	338	+
	339	+ result->peer_tq_xdp_xmit_err = 0;
	340	+ result->xdp_packets = 0;
	341	+ result->xdp_tx_err = 0;
	342	+ result->xdp_bytes = 0;
	343	+ result->rx_drops = 0;
	344	+ for (i = 0; i < dev->num_rx_queues; i++) {
	345	+ u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err;
	346	+ struct veth_rq_stats *stats = &priv->rq[i].stats;
233	347	unsigned int start;
234	348
235	349	do {
236	350	start = u64_stats_fetch_begin_irq(&stats->syncp);
237		- packets = stats->packets;
238		- bytes = stats->bytes;
	351	+ peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err;
	352	+ xdp_tx_err = stats->vs.xdp_tx_err;
	353	+ packets = stats->vs.xdp_packets;
	354	+ bytes = stats->vs.xdp_bytes;
	355	+ drops = stats->vs.rx_drops;
239	356	} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
240		- result->packets += packets;
241		- result->bytes += bytes;
	357	+ result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err;
	358	+ result->xdp_tx_err += xdp_tx_err;
	359	+ result->xdp_packets += packets;
	360	+ result->xdp_bytes += bytes;
	361	+ result->rx_drops += drops;
242	362	}
243		- return atomic64_read(&priv->dropped);
244	363	}
245	364
246	365	static void veth_get_stats64(struct net_device *dev,
..	..	@@ -248,18 +367,31 @@
248	367	{
249	368	struct veth_priv *priv = netdev_priv(dev);
250	369	struct net_device *peer;
251		- struct pcpu_vstats one;
	370	+ struct veth_stats rx;
	371	+ u64 packets, bytes;
252	372
253		- tot->tx_dropped = veth_stats_one(&one, dev);
254		- tot->tx_bytes = one.bytes;
255		- tot->tx_packets = one.packets;
	373	+ tot->tx_dropped = veth_stats_tx(dev, &packets, &bytes);
	374	+ tot->tx_bytes = bytes;
	375	+ tot->tx_packets = packets;
	376	+
	377	+ veth_stats_rx(&rx, dev);
	378	+ tot->tx_dropped += rx.xdp_tx_err;
	379	+ tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err;
	380	+ tot->rx_bytes = rx.xdp_bytes;
	381	+ tot->rx_packets = rx.xdp_packets;
256	382
257	383	rcu_read_lock();
258	384	peer = rcu_dereference(priv->peer);
259	385	if (peer) {
260		- tot->rx_dropped = veth_stats_one(&one, peer);
261		- tot->rx_bytes = one.bytes;
262		- tot->rx_packets = one.packets;
	386	+ veth_stats_tx(peer, &packets, &bytes);
	387	+ tot->rx_bytes += bytes;
	388	+ tot->rx_packets += packets;
	389	+
	390	+ veth_stats_rx(&rx, peer);
	391	+ tot->tx_dropped += rx.peer_tq_xdp_xmit_err;
	392	+ tot->rx_dropped += rx.xdp_tx_err;
	393	+ tot->tx_bytes += rx.xdp_bytes;
	394	+ tot->tx_packets += rx.xdp_packets;
263	395	}
264	396	rcu_read_unlock();
265	397	}
..	..	@@ -274,10 +406,6 @@
274	406	{
275	407	struct sk_buff *skb;
276	408
277		- if (!buflen) {
278		- buflen = SKB_DATA_ALIGN(headroom + len) +
279		- SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
280		- }
281	409	skb = build_skb(head, buflen);
282	410	if (!skb)
283	411	return NULL;
..	..	@@ -293,21 +421,31 @@
293	421	return smp_processor_id() % dev->real_num_rx_queues;
294	422	}
295	423
	424	+static struct net_device veth_peer_dev(struct net_device dev)
	425	+{
	426	+ struct veth_priv *priv = netdev_priv(dev);
	427	+
	428	+ /* Callers must be under RCU read side. */
	429	+ return rcu_dereference(priv->peer);
	430	+}
	431	+
296	432	static int veth_xdp_xmit(struct net_device *dev, int n,
297		- struct xdp_frame **frames, u32 flags)
	433	+ struct xdp_frame **frames,
	434	+ u32 flags, bool ndo_xmit)
298	435	{
299	436	struct veth_priv rcv_priv, priv = netdev_priv(dev);
	437	+ int i, ret = -ENXIO, drops = 0;
300	438	struct net_device *rcv;
301	439	unsigned int max_len;
302	440	struct veth_rq *rq;
303		- int i, drops = 0;
304	441
305	442	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
306	443	return -EINVAL;
307	444
	445	+ rcu_read_lock();
308	446	rcv = rcu_dereference(priv->peer);
309	447	if (unlikely(!rcv))
310		- return -ENXIO;
	448	+ goto out;
311	449
312	450	rcv_priv = netdev_priv(rcv);
313	451	rq = &rcv_priv->rq[veth_select_rxq(rcv)];
..	..	@@ -316,7 +454,7 @@
316	454	* device is up.
317	455	*/
318	456	if (!rcu_access_pointer(rq->xdp_prog))
319		- return -ENXIO;
	457	+ goto out;
320	458
321	459	max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN;
322	460
..	..	@@ -336,44 +474,99 @@
336	474	if (flags & XDP_XMIT_FLUSH)
337	475	__veth_xdp_flush(rq);
338	476
339		- return n - drops;
	477	+ ret = n - drops;
	478	+ if (ndo_xmit) {
	479	+ u64_stats_update_begin(&rq->stats.syncp);
	480	+ rq->stats.vs.peer_tq_xdp_xmit += n - drops;
	481	+ rq->stats.vs.peer_tq_xdp_xmit_err += drops;
	482	+ u64_stats_update_end(&rq->stats.syncp);
	483	+ }
	484	+
	485	+out:
	486	+ rcu_read_unlock();
	487	+
	488	+ return ret;
340	489	}
341	490
342		-static void veth_xdp_flush(struct net_device *dev)
	491	+static int veth_ndo_xdp_xmit(struct net_device *dev, int n,
	492	+ struct xdp_frame **frames, u32 flags)
343	493	{
344		- struct veth_priv rcv_priv, priv = netdev_priv(dev);
	494	+ int err;
	495	+
	496	+ err = veth_xdp_xmit(dev, n, frames, flags, true);
	497	+ if (err < 0) {
	498	+ struct veth_priv *priv = netdev_priv(dev);
	499	+
	500	+ atomic64_add(n, &priv->dropped);
	501	+ }
	502	+
	503	+ return err;
	504	+}
	505	+
	506	+static void veth_xdp_flush_bq(struct veth_rq rq, struct veth_xdp_tx_bq bq)
	507	+{
	508	+ int sent, i, err = 0;
	509	+
	510	+ sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false);
	511	+ if (sent < 0) {
	512	+ err = sent;
	513	+ sent = 0;
	514	+ for (i = 0; i < bq->count; i++)
	515	+ xdp_return_frame(bq->q[i]);
	516	+ }
	517	+ trace_xdp_bulk_tx(rq->dev, sent, bq->count - sent, err);
	518	+
	519	+ u64_stats_update_begin(&rq->stats.syncp);
	520	+ rq->stats.vs.xdp_tx += sent;
	521	+ rq->stats.vs.xdp_tx_err += bq->count - sent;
	522	+ u64_stats_update_end(&rq->stats.syncp);
	523	+
	524	+ bq->count = 0;
	525	+}
	526	+
	527	+static void veth_xdp_flush(struct veth_rq rq, struct veth_xdp_tx_bq bq)
	528	+{
	529	+ struct veth_priv rcv_priv, priv = netdev_priv(rq->dev);
345	530	struct net_device *rcv;
346		- struct veth_rq *rq;
	531	+ struct veth_rq *rcv_rq;
347	532
348	533	rcu_read_lock();
	534	+ veth_xdp_flush_bq(rq, bq);
349	535	rcv = rcu_dereference(priv->peer);
350	536	if (unlikely(!rcv))
351	537	goto out;
352	538
353	539	rcv_priv = netdev_priv(rcv);
354		- rq = &rcv_priv->rq[veth_select_rxq(rcv)];
	540	+ rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)];
355	541	/* xdp_ring is initialized on receive side? */
356		- if (unlikely(!rcu_access_pointer(rq->xdp_prog)))
	542	+ if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog)))
357	543	goto out;
358	544
359		- __veth_xdp_flush(rq);
	545	+ __veth_xdp_flush(rcv_rq);
360	546	out:
361	547	rcu_read_unlock();
362	548	}
363	549
364		-static int veth_xdp_tx(struct net_device dev, struct xdp_buff xdp)
	550	+static int veth_xdp_tx(struct veth_rq rq, struct xdp_buff xdp,
	551	+ struct veth_xdp_tx_bq *bq)
365	552	{
366		- struct xdp_frame *frame = convert_to_xdp_frame(xdp);
	553	+ struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp);
367	554
368	555	if (unlikely(!frame))
369	556	return -EOVERFLOW;
370	557
371		- return veth_xdp_xmit(dev, 1, &frame, 0);
	558	+ if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE))
	559	+ veth_xdp_flush_bq(rq, bq);
	560	+
	561	+ bq->q[bq->count++] = frame;
	562	+
	563	+ return 0;
372	564	}
373	565
374	566	static struct sk_buff veth_xdp_rcv_one(struct veth_rq rq,
375	567	struct xdp_frame *frame,
376		- unsigned int *xdp_xmit)
	568	+ struct veth_xdp_tx_bq *bq,
	569	+ struct veth_stats *stats)
377	570	{
378	571	void *hard_start = frame->data - frame->headroom;
379	572	int len = frame->len, delta = 0;
..	..	@@ -391,10 +584,7 @@
391	584	struct xdp_buff xdp;
392	585	u32 act;
393	586
394		- xdp.data_hard_start = hard_start;
395		- xdp.data = frame->data;
396		- xdp.data_end = frame->data + frame->len;
397		- xdp.data_meta = frame->data - frame->metasize;
	587	+ xdp_convert_frame_to_buff(frame, &xdp);
398	588	xdp.rxq = &rq->xdp_rxq;
399	589
400	590	act = bpf_prog_run_xdp(xdp_prog, &xdp);
..	..	@@ -407,12 +597,13 @@
407	597	case XDP_TX:
408	598	orig_frame = *frame;
409	599	xdp.rxq->mem = frame->mem;
410		- if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) {
	600	+ if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) {
411	601	trace_xdp_exception(rq->dev, xdp_prog, act);
412	602	frame = &orig_frame;
	603	+ stats->rx_drops++;
413	604	goto err_xdp;
414	605	}
415		- *xdp_xmit \|= VETH_XDP_TX;
	606	+ stats->xdp_tx++;
416	607	rcu_read_unlock();
417	608	goto xdp_xmit;
418	609	case XDP_REDIRECT:
..	..	@@ -420,28 +611,34 @@
420	611	xdp.rxq->mem = frame->mem;
421	612	if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) {
422	613	frame = &orig_frame;
	614	+ stats->rx_drops++;
423	615	goto err_xdp;
424	616	}
425		- *xdp_xmit \|= VETH_XDP_REDIR;
	617	+ stats->xdp_redirect++;
426	618	rcu_read_unlock();
427	619	goto xdp_xmit;
428	620	default:
429	621	bpf_warn_invalid_xdp_action(act);
	622	+ fallthrough;
430	623	case XDP_ABORTED:
431	624	trace_xdp_exception(rq->dev, xdp_prog, act);
	625	+ fallthrough;
432	626	case XDP_DROP:
	627	+ stats->xdp_drops++;
433	628	goto err_xdp;
434	629	}
435	630	}
436	631	rcu_read_unlock();
437	632
438	633	headroom = sizeof(struct xdp_frame) + frame->headroom - delta;
439		- skb = veth_build_skb(hard_start, headroom, len, 0);
	634	+ skb = veth_build_skb(hard_start, headroom, len, frame->frame_sz);
440	635	if (!skb) {
441	636	xdp_return_frame(frame);
	637	+ stats->rx_drops++;
442	638	goto err;
443	639	}
444	640
	641	+ xdp_release_frame(frame);
445	642	xdp_scrub_frame(frame);
446	643	skb->protocol = eth_type_trans(skb, rq->dev);
447	644	err:
..	..	@@ -453,8 +650,10 @@
453	650	return NULL;
454	651	}
455	652
456		-static struct sk_buff veth_xdp_rcv_skb(struct veth_rq rq, struct sk_buff *skb,
457		- unsigned int *xdp_xmit)
	653	+static struct sk_buff veth_xdp_rcv_skb(struct veth_rq rq,
	654	+ struct sk_buff *skb,
	655	+ struct veth_xdp_tx_bq *bq,
	656	+ struct veth_stats *stats)
458	657	{
459	658	u32 pktlen, headroom, act, metalen;
460	659	void orig_data, orig_data_end;
..	..	@@ -498,9 +697,8 @@
498	697	goto drop;
499	698	}
500	699
501		- nskb = veth_build_skb(head,
502		- VETH_XDP_HEADROOM + mac_len, skb->len,
503		- PAGE_SIZE);
	700	+ nskb = veth_build_skb(head, VETH_XDP_HEADROOM + mac_len,
	701	+ skb->len, PAGE_SIZE);
504	702	if (!nskb) {
505	703	page_frag_free(head);
506	704	goto drop;
..	..	@@ -518,6 +716,11 @@
518	716	xdp.data_end = xdp.data + pktlen;
519	717	xdp.data_meta = xdp.data;
520	718	xdp.rxq = &rq->xdp_rxq;
	719	+
	720	+ /* SKB "head" area always have tailroom for skb_shared_info */
	721	+ xdp.frame_sz = (void *)skb_end_pointer(skb) - xdp.data_hard_start;
	722	+ xdp.frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	723	+
521	724	orig_data = xdp.data;
522	725	orig_data_end = xdp.data_end;
523	726
..	..	@@ -530,31 +733,38 @@
530	733	get_page(virt_to_page(xdp.data));
531	734	consume_skb(skb);
532	735	xdp.rxq->mem = rq->xdp_mem;
533		- if (unlikely(veth_xdp_tx(rq->dev, &xdp) < 0)) {
	736	+ if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) {
534	737	trace_xdp_exception(rq->dev, xdp_prog, act);
	738	+ stats->rx_drops++;
535	739	goto err_xdp;
536	740	}
537		- *xdp_xmit \|= VETH_XDP_TX;
	741	+ stats->xdp_tx++;
538	742	rcu_read_unlock();
539	743	goto xdp_xmit;
540	744	case XDP_REDIRECT:
541	745	get_page(virt_to_page(xdp.data));
542	746	consume_skb(skb);
543	747	xdp.rxq->mem = rq->xdp_mem;
544		- if (xdp_do_redirect(rq->dev, &xdp, xdp_prog))
	748	+ if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) {
	749	+ stats->rx_drops++;
545	750	goto err_xdp;
546		- *xdp_xmit \|= VETH_XDP_REDIR;
	751	+ }
	752	+ stats->xdp_redirect++;
547	753	rcu_read_unlock();
548	754	goto xdp_xmit;
549	755	default:
550	756	bpf_warn_invalid_xdp_action(act);
	757	+ fallthrough;
551	758	case XDP_ABORTED:
552	759	trace_xdp_exception(rq->dev, xdp_prog, act);
	760	+ fallthrough;
553	761	case XDP_DROP:
554		- goto drop;
	762	+ stats->xdp_drops++;
	763	+ goto xdp_drop;
555	764	}
556	765	rcu_read_unlock();
557	766
	767	+ /* check if bpf_xdp_adjust_head was used */
558	768	delta = orig_data - xdp.data;
559	769	off = mac_len + delta;
560	770	if (off > 0)
..	..	@@ -562,9 +772,11 @@
562	772	else if (off < 0)
563	773	__skb_pull(skb, -off);
564	774	skb->mac_header -= delta;
	775	+
	776	+ /* check if bpf_xdp_adjust_tail was used */
565	777	off = xdp.data_end - orig_data_end;
566	778	if (off != 0)
567		- __skb_put(skb, off);
	779	+ __skb_put(skb, off); /* positive on grow, negative on shrink */
568	780	skb->protocol = eth_type_trans(skb, rq->dev);
569	781
570	782	metalen = xdp.data - xdp.data_meta;
..	..	@@ -573,6 +785,8 @@
573	785	out:
574	786	return skb;
575	787	drop:
	788	+ stats->rx_drops++;
	789	+xdp_drop:
576	790	rcu_read_unlock();
577	791	kfree_skb(skb);
578	792	return NULL;
..	..	@@ -583,7 +797,9 @@
583	797	return NULL;
584	798	}
585	799
586		-static int veth_xdp_rcv(struct veth_rq rq, int budget, unsigned int xdp_xmit)
	800	+static int veth_xdp_rcv(struct veth_rq *rq, int budget,
	801	+ struct veth_xdp_tx_bq *bq,
	802	+ struct veth_stats *stats)
587	803	{
588	804	int i, done = 0;
589	805
..	..	@@ -595,10 +811,14 @@
595	811	break;
596	812
597	813	if (veth_is_xdp_frame(ptr)) {
598		- skb = veth_xdp_rcv_one(rq, veth_ptr_to_xdp(ptr),
599		- xdp_xmit);
	814	+ struct xdp_frame *frame = veth_ptr_to_xdp(ptr);
	815	+
	816	+ stats->xdp_bytes += frame->len;
	817	+ skb = veth_xdp_rcv_one(rq, frame, bq, stats);
600	818	} else {
601		- skb = veth_xdp_rcv_skb(rq, ptr, xdp_xmit);
	819	+ skb = ptr;
	820	+ stats->xdp_bytes += skb->len;
	821	+ skb = veth_xdp_rcv_skb(rq, skb, bq, stats);
602	822	}
603	823
604	824	if (skb)
..	..	@@ -607,6 +827,14 @@
607	827	done++;
608	828	}
609	829
	830	+ u64_stats_update_begin(&rq->stats.syncp);
	831	+ rq->stats.vs.xdp_redirect += stats->xdp_redirect;
	832	+ rq->stats.vs.xdp_bytes += stats->xdp_bytes;
	833	+ rq->stats.vs.xdp_drops += stats->xdp_drops;
	834	+ rq->stats.vs.rx_drops += stats->rx_drops;
	835	+ rq->stats.vs.xdp_packets += done;
	836	+ u64_stats_update_end(&rq->stats.syncp);
	837	+
610	838	return done;
611	839	}
612	840
..	..	@@ -614,11 +842,17 @@
614	842	{
615	843	struct veth_rq *rq =
616	844	container_of(napi, struct veth_rq, xdp_napi);
617		- unsigned int xdp_xmit = 0;
	845	+ struct veth_stats stats = {};
	846	+ struct veth_xdp_tx_bq bq;
618	847	int done;
619	848
	849	+ bq.count = 0;
	850	+
620	851	xdp_set_return_frame_no_direct();
621		- done = veth_xdp_rcv(rq, budget, &xdp_xmit);
	852	+ done = veth_xdp_rcv(rq, budget, &bq, &stats);
	853	+
	854	+ if (stats.xdp_redirect > 0)
	855	+ xdp_do_flush();
622	856
623	857	if (done < budget && napi_complete_done(napi, done)) {
624	858	/* Write rx_notify_masked before reading ptr_ring */
..	..	@@ -631,10 +865,8 @@
631	865	}
632	866	}
633	867
634		- if (xdp_xmit & VETH_XDP_TX)
635		- veth_xdp_flush(rq->dev);
636		- if (xdp_xmit & VETH_XDP_REDIR)
637		- xdp_do_flush_map();
	868	+ if (stats.xdp_tx > 0)
	869	+ veth_xdp_flush(rq, &bq);
638	870	xdp_clear_return_frame_no_direct();
639	871
640	872	return done;
..	..	@@ -677,14 +909,13 @@
677	909	struct veth_rq *rq = &priv->rq[i];
678	910
679	911	napi_disable(&rq->xdp_napi);
680		- napi_hash_del(&rq->xdp_napi);
	912	+ __netif_napi_del(&rq->xdp_napi);
681	913	}
682	914	synchronize_net();
683	915
684	916	for (i = 0; i < dev->real_num_rx_queues; i++) {
685	917	struct veth_rq *rq = &priv->rq[i];
686	918
687		- netif_napi_del(&rq->xdp_napi);
688	919	rq->rx_notify_masked = false;
689	920	ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free);
690	921	}
..	..	@@ -799,8 +1030,10 @@
799	1030	if (!priv->rq)
800	1031	return -ENOMEM;
801	1032
802		- for (i = 0; i < dev->num_rx_queues; i++)
	1033	+ for (i = 0; i < dev->num_rx_queues; i++) {
803	1034	priv->rq[i].dev = dev;
	1035	+ u64_stats_init(&priv->rq[i].stats.syncp);
	1036	+ }
804	1037
805	1038	return 0;
806	1039	}
..	..	@@ -816,13 +1049,13 @@
816	1049	{
817	1050	int err;
818	1051
819		- dev->vstats = netdev_alloc_pcpu_stats(struct pcpu_vstats);
820		- if (!dev->vstats)
	1052	+ dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
	1053	+ if (!dev->lstats)
821	1054	return -ENOMEM;
822	1055
823	1056	err = veth_alloc_queues(dev);
824	1057	if (err) {
825		- free_percpu(dev->vstats);
	1058	+ free_percpu(dev->lstats);
826	1059	return err;
827	1060	}
828	1061
..	..	@@ -832,7 +1065,7 @@
832	1065	static void veth_dev_free(struct net_device *dev)
833	1066	{
834	1067	veth_free_queues(dev);
835		- free_percpu(dev->vstats);
	1068	+ free_percpu(dev->lstats);
836	1069	}
837	1070
838	1071	#ifdef CONFIG_NET_POLL_CONTROLLER
..	..	@@ -976,26 +1209,11 @@
976	1209	return err;
977	1210	}
978	1211
979		-static u32 veth_xdp_query(struct net_device *dev)
980		-{
981		- struct veth_priv *priv = netdev_priv(dev);
982		- const struct bpf_prog *xdp_prog;
983		-
984		- xdp_prog = priv->_xdp_prog;
985		- if (xdp_prog)
986		- return xdp_prog->aux->id;
987		-
988		- return 0;
989		-}
990		-
991	1212	static int veth_xdp(struct net_device dev, struct netdev_bpf xdp)
992	1213	{
993	1214	switch (xdp->command) {
994	1215	case XDP_SETUP_PROG:
995	1216	return veth_xdp_set(dev, xdp->prog, xdp->extack);
996		- case XDP_QUERY_PROG:
997		- xdp->prog_id = veth_xdp_query(dev);
998		- return 0;
999	1217	default:
1000	1218	return -EINVAL;
1001	1219	}
..	..	@@ -1017,7 +1235,8 @@
1017	1235	.ndo_features_check = passthru_features_check,
1018	1236	.ndo_set_rx_headroom = veth_set_rx_headroom,
1019	1237	.ndo_bpf = veth_xdp,
1020		- .ndo_xdp_xmit = veth_xdp_xmit,
	1238	+ .ndo_xdp_xmit = veth_ndo_xdp_xmit,
	1239	+ .ndo_get_peer_dev = veth_peer_dev,
1021	1240	};
1022	1241
1023	1242	#define VETH_FEATURES (NETIF_F_SG \| NETIF_F_FRAGLIST \| NETIF_F_HW_CSUM \| \
..	..	@@ -1096,10 +1315,7 @@
1096	1315
1097	1316	nla_peer = data[VETH_INFO_PEER];
1098	1317	ifmp = nla_data(nla_peer);
1099		- err = rtnl_nla_parse_ifla(peer_tb,
1100		- nla_data(nla_peer) + sizeof(struct ifinfomsg),
1101		- nla_len(nla_peer) - sizeof(struct ifinfomsg),
1102		- NULL);
	1318	+ err = rtnl_nla_parse_ifinfomsg(peer_tb, nla_peer, extack);
1103	1319	if (err < 0)
1104	1320	return err;
1105	1321
..	..	@@ -1126,7 +1342,7 @@
1126	1342	return PTR_ERR(net);
1127	1343
1128	1344	peer = rtnl_create_link(net, ifname, name_assign_type,
1129		- &veth_link_ops, tbp);
	1345	+ &veth_link_ops, tbp, extack);
1130	1346	if (IS_ERR(peer)) {
1131	1347	put_net(net);
1132	1348	return PTR_ERR(peer);