forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/tools/testing/selftests/net/udpgso_bench_tx.c
....@@ -5,6 +5,8 @@
55 #include <arpa/inet.h>
66 #include <errno.h>
77 #include <error.h>
8
+#include <linux/errqueue.h>
9
+#include <linux/net_tstamp.h>
810 #include <netinet/if_ether.h>
911 #include <netinet/in.h>
1012 #include <netinet/ip.h>
....@@ -19,8 +21,11 @@
1921 #include <string.h>
2022 #include <sys/socket.h>
2123 #include <sys/time.h>
24
+#include <sys/poll.h>
2225 #include <sys/types.h>
2326 #include <unistd.h>
27
+
28
+#include "../kselftest.h"
2429 
2530 #ifndef ETH_MAX_MTU
2631 #define ETH_MAX_MTU 0xFFFFU
....@@ -34,8 +39,16 @@
3439 #define SO_ZEROCOPY	60
3540 #endif
3641 
42
+#ifndef SO_EE_ORIGIN_ZEROCOPY
43
+#define SO_EE_ORIGIN_ZEROCOPY 5
44
+#endif
45
+
3746 #ifndef MSG_ZEROCOPY
3847 #define MSG_ZEROCOPY	0x4000000
48
+#endif
49
+
50
+#ifndef ENOTSUPP
51
+#define ENOTSUPP	524
3952 #endif
4053 
4154 #define NUM_PKT		100
....@@ -48,10 +61,24 @@
4861 static int	cfg_payload_len	= (1472 * 42);
4962 static int	cfg_port	= 8000;
5063 static int	cfg_runtime_ms	= -1;
64
+static bool	cfg_poll;
5165 static bool	cfg_segment;
5266 static bool	cfg_sendmmsg;
5367 static bool	cfg_tcp;
68
+static uint32_t	cfg_tx_ts = SOF_TIMESTAMPING_TX_SOFTWARE;
69
+static bool	cfg_tx_tstamp;
70
+static bool	cfg_audit;
71
+static bool	cfg_verbose;
5472 static bool	cfg_zerocopy;
73
+static int	cfg_msg_nr;
74
+static uint16_t	cfg_gso_size;
75
+static unsigned long total_num_msgs;
76
+static unsigned long total_num_sends;
77
+static unsigned long stat_tx_ts;
78
+static unsigned long stat_tx_ts_errors;
79
+static unsigned long tstart;
80
+static unsigned long tend;
81
+static unsigned long stat_zcopies;
5582 
5683 static socklen_t cfg_alen;
5784 static struct sockaddr_storage cfg_dst_addr;
....@@ -108,21 +135,123 @@
108135 	}
109136 }
110137 
111
-static void flush_zerocopy(int fd)
138
+static void flush_cmsg(struct cmsghdr *cmsg)
112139 {
113
-	struct msghdr msg = {0};	/* flush */
140
+	struct sock_extended_err *err;
141
+	struct scm_timestamping *tss;
142
+	__u32 lo;
143
+	__u32 hi;
144
+	int i;
145
+
146
+	switch (cmsg->cmsg_level) {
147
+	case SOL_SOCKET:
148
+		if (cmsg->cmsg_type == SO_TIMESTAMPING) {
149
+			i = (cfg_tx_ts == SOF_TIMESTAMPING_TX_HARDWARE) ? 2 : 0;
150
+			tss = (struct scm_timestamping *)CMSG_DATA(cmsg);
151
+			if (tss->ts[i].tv_sec == 0)
152
+				stat_tx_ts_errors++;
153
+		} else {
154
+			error(1, 0, "unknown SOL_SOCKET cmsg type=%u\n",
155
+			      cmsg->cmsg_type);
156
+		}
157
+		break;
158
+	case SOL_IP:
159
+	case SOL_IPV6:
160
+		switch (cmsg->cmsg_type) {
161
+		case IP_RECVERR:
162
+		case IPV6_RECVERR:
163
+		{
164
+			err = (struct sock_extended_err *)CMSG_DATA(cmsg);
165
+			switch (err->ee_origin) {
166
+			case SO_EE_ORIGIN_TIMESTAMPING:
167
+				/* Got a TX timestamp from error queue */
168
+				stat_tx_ts++;
169
+				break;
170
+			case SO_EE_ORIGIN_ICMP:
171
+			case SO_EE_ORIGIN_ICMP6:
172
+				if (cfg_verbose)
173
+					fprintf(stderr,
174
+						"received ICMP error: type=%u, code=%u\n",
175
+						err->ee_type, err->ee_code);
176
+				break;
177
+			case SO_EE_ORIGIN_ZEROCOPY:
178
+			{
179
+				lo = err->ee_info;
180
+				hi = err->ee_data;
181
+				/* range of IDs acknowledged */
182
+				stat_zcopies += hi - lo + 1;
183
+				break;
184
+			}
185
+			case SO_EE_ORIGIN_LOCAL:
186
+				if (cfg_verbose)
187
+					fprintf(stderr,
188
+						"received packet with local origin: %u\n",
189
+						err->ee_origin);
190
+				break;
191
+			default:
192
+				error(0, 1, "received packet with origin: %u",
193
+				      err->ee_origin);
194
+			}
195
+			break;
196
+		}
197
+		default:
198
+			error(0, 1, "unknown IP msg type=%u\n",
199
+			      cmsg->cmsg_type);
200
+			break;
201
+		}
202
+		break;
203
+	default:
204
+		error(0, 1, "unknown cmsg level=%u\n",
205
+		      cmsg->cmsg_level);
206
+	}
207
+}
208
+
209
+static void flush_errqueue_recv(int fd)
210
+{
211
+	char control[CMSG_SPACE(sizeof(struct scm_timestamping)) +
212
+		     CMSG_SPACE(sizeof(struct sock_extended_err)) +
213
+		     CMSG_SPACE(sizeof(struct sockaddr_in6))] = {0};
214
+	struct msghdr msg = {0};
215
+	struct cmsghdr *cmsg;
114216 	int ret;
115217 
116218 	while (1) {
219
+		msg.msg_control = control;
220
+		msg.msg_controllen = sizeof(control);
117221 		ret = recvmsg(fd, &msg, MSG_ERRQUEUE);
118222 		if (ret == -1 && errno == EAGAIN)
119223 			break;
120224 		if (ret == -1)
121225 			error(1, errno, "errqueue");
122
-		if (msg.msg_flags != (MSG_ERRQUEUE | MSG_CTRUNC))
226
+		if (msg.msg_flags != MSG_ERRQUEUE)
123227 			error(1, 0, "errqueue: flags 0x%x\n", msg.msg_flags);
228
+		if (cfg_audit) {
229
+			for (cmsg = CMSG_FIRSTHDR(&msg);
230
+					cmsg;
231
+					cmsg = CMSG_NXTHDR(&msg, cmsg))
232
+				flush_cmsg(cmsg);
233
+		}
124234 		msg.msg_flags = 0;
125235 	}
236
+}
237
+
238
+static void flush_errqueue(int fd, const bool do_poll)
239
+{
240
+	if (do_poll) {
241
+		struct pollfd fds = {0};
242
+		int ret;
243
+
244
+		fds.fd = fd;
245
+		ret = poll(&fds, 1, 500);
246
+		if (ret == 0) {
247
+			if (cfg_verbose)
248
+				fprintf(stderr, "poll timeout\n");
249
+		} else if (ret < 0) {
250
+			error(1, errno, "poll");
251
+		}
252
+	}
253
+
254
+	flush_errqueue_recv(fd);
126255 }
127256 
128257 static int send_tcp(int fd, char *data)
....@@ -166,15 +295,39 @@
166295 	return count;
167296 }
168297 
298
+static void send_ts_cmsg(struct cmsghdr *cm)
299
+{
300
+	uint32_t *valp;
301
+
302
+	cm->cmsg_level = SOL_SOCKET;
303
+	cm->cmsg_type = SO_TIMESTAMPING;
304
+	cm->cmsg_len = CMSG_LEN(sizeof(cfg_tx_ts));
305
+	valp = (void *)CMSG_DATA(cm);
306
+	*valp = cfg_tx_ts;
307
+}
308
+
169309 static int send_udp_sendmmsg(int fd, char *data)
170310 {
311
+	char control[CMSG_SPACE(sizeof(cfg_tx_ts))] = {0};
171312 	const int max_nr_msg = ETH_MAX_MTU / ETH_DATA_LEN;
172313 	struct mmsghdr mmsgs[max_nr_msg];
173314 	struct iovec iov[max_nr_msg];
174315 	unsigned int off = 0, left;
316
+	size_t msg_controllen = 0;
175317 	int i = 0, ret;
176318 
177319 	memset(mmsgs, 0, sizeof(mmsgs));
320
+
321
+	if (cfg_tx_tstamp) {
322
+		struct msghdr msg = {0};
323
+		struct cmsghdr *cmsg;
324
+
325
+		msg.msg_control = control;
326
+		msg.msg_controllen = sizeof(control);
327
+		cmsg = CMSG_FIRSTHDR(&msg);
328
+		send_ts_cmsg(cmsg);
329
+		msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts));
330
+	}
178331 
179332 	left = cfg_payload_len;
180333 	while (left) {
....@@ -186,6 +339,13 @@
186339 
187340 		mmsgs[i].msg_hdr.msg_iov = iov + i;
188341 		mmsgs[i].msg_hdr.msg_iovlen = 1;
342
+
343
+		mmsgs[i].msg_hdr.msg_name = (void *)&cfg_dst_addr;
344
+		mmsgs[i].msg_hdr.msg_namelen = cfg_alen;
345
+		if (msg_controllen) {
346
+			mmsgs[i].msg_hdr.msg_control = control;
347
+			mmsgs[i].msg_hdr.msg_controllen = msg_controllen;
348
+		}
189349 
190350 		off += iov[i].iov_len;
191351 		left -= iov[i].iov_len;
....@@ -205,16 +365,19 @@
205365 
206366 	cm->cmsg_level = SOL_UDP;
207367 	cm->cmsg_type = UDP_SEGMENT;
208
-	cm->cmsg_len = CMSG_LEN(sizeof(cfg_mss));
368
+	cm->cmsg_len = CMSG_LEN(sizeof(cfg_gso_size));
209369 	valp = (void *)CMSG_DATA(cm);
210
-	*valp = cfg_mss;
370
+	*valp = cfg_gso_size;
211371 }
212372 
213373 static int send_udp_segment(int fd, char *data)
214374 {
215
-	char control[CMSG_SPACE(sizeof(cfg_mss))] = {0};
375
+	char control[CMSG_SPACE(sizeof(cfg_gso_size)) +
376
+		     CMSG_SPACE(sizeof(cfg_tx_ts))] = {0};
216377 	struct msghdr msg = {0};
217378 	struct iovec iov = {0};
379
+	size_t msg_controllen;
380
+	struct cmsghdr *cmsg;
218381 	int ret;
219382 
220383 	iov.iov_base = data;
....@@ -225,8 +388,16 @@
225388 
226389 	msg.msg_control = control;
227390 	msg.msg_controllen = sizeof(control);
228
-	send_udp_segment_cmsg(CMSG_FIRSTHDR(&msg));
391
+	cmsg = CMSG_FIRSTHDR(&msg);
392
+	send_udp_segment_cmsg(cmsg);
393
+	msg_controllen = CMSG_SPACE(sizeof(cfg_mss));
394
+	if (cfg_tx_tstamp) {
395
+		cmsg = CMSG_NXTHDR(&msg, cmsg);
396
+		send_ts_cmsg(cmsg);
397
+		msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts));
398
+	}
229399 
400
+	msg.msg_controllen = msg_controllen;
230401 	msg.msg_name = (void *)&cfg_dst_addr;
231402 	msg.msg_namelen = cfg_alen;
232403 
....@@ -234,14 +405,15 @@
234405 	if (ret == -1)
235406 		error(1, errno, "sendmsg");
236407 	if (ret != iov.iov_len)
237
-		error(1, 0, "sendmsg: %u != %lu\n", ret, iov.iov_len);
408
+		error(1, 0, "sendmsg: %u != %llu\n", ret,
409
+			(unsigned long long)iov.iov_len);
238410 
239411 	return 1;
240412 }
241413 
242414 static void usage(const char *filepath)
243415 {
244
-	error(1, 0, "Usage: %s [-46cmStuz] [-C cpu] [-D dst ip] [-l secs] [-p port] [-s sendsize]",
416
+	error(1, 0, "Usage: %s [-46acmHPtTuvz] [-C cpu] [-D dst ip] [-l secs] [-M messagenr] [-p port] [-s sendsize] [-S gsosize]",
245417 		    filepath);
246418 }
247419 
....@@ -251,7 +423,7 @@
251423 	int max_len, hdrlen;
252424 	int c;
253425 
254
-	while ((c = getopt(argc, argv, "46cC:D:l:mp:s:Stuz")) != -1) {
426
+	while ((c = getopt(argc, argv, "46acC:D:Hl:mM:p:s:PS:tTuvz")) != -1) {
255427 		switch (c) {
256428 		case '4':
257429 			if (cfg_family != PF_UNSPEC)
....@@ -264,6 +436,9 @@
264436 				error(1, 0, "Pass one of -4 or -6");
265437 			cfg_family = PF_INET6;
266438 			cfg_alen = sizeof(struct sockaddr_in6);
439
+			break;
440
+		case 'a':
441
+			cfg_audit = true;
267442 			break;
268443 		case 'c':
269444 			cfg_cache_trash = true;
....@@ -280,20 +455,37 @@
280455 		case 'm':
281456 			cfg_sendmmsg = true;
282457 			break;
458
+		case 'M':
459
+			cfg_msg_nr = strtoul(optarg, NULL, 10);
460
+			break;
283461 		case 'p':
284462 			cfg_port = strtoul(optarg, NULL, 0);
463
+			break;
464
+		case 'P':
465
+			cfg_poll = true;
285466 			break;
286467 		case 's':
287468 			cfg_payload_len = strtoul(optarg, NULL, 0);
288469 			break;
289470 		case 'S':
471
+			cfg_gso_size = strtoul(optarg, NULL, 0);
290472 			cfg_segment = true;
473
+			break;
474
+		case 'H':
475
+			cfg_tx_ts = SOF_TIMESTAMPING_TX_HARDWARE;
476
+			cfg_tx_tstamp = true;
291477 			break;
292478 		case 't':
293479 			cfg_tcp = true;
294480 			break;
481
+		case 'T':
482
+			cfg_tx_tstamp = true;
483
+			break;
295484 		case 'u':
296485 			cfg_connected = false;
486
+			break;
487
+		case 'v':
488
+			cfg_verbose = true;
297489 			break;
298490 		case 'z':
299491 			cfg_zerocopy = true;
....@@ -315,6 +507,8 @@
315507 		error(1, 0, "connectionless tcp makes no sense");
316508 	if (cfg_segment && cfg_sendmmsg)
317509 		error(1, 0, "cannot combine segment offload and sendmmsg");
510
+	if (cfg_tx_tstamp && !(cfg_segment || cfg_sendmmsg))
511
+		error(1, 0, "Options -T and -H require either -S or -m option");
318512 
319513 	if (cfg_family == PF_INET)
320514 		hdrlen = sizeof(struct iphdr) + sizeof(struct udphdr);
....@@ -323,6 +517,8 @@
323517 
324518 	cfg_mss = ETH_DATA_LEN - hdrlen;
325519 	max_len = ETH_MAX_MTU - hdrlen;
520
+	if (!cfg_gso_size)
521
+		cfg_gso_size = cfg_mss;
326522 
327523 	if (cfg_payload_len > max_len)
328524 		error(1, 0, "payload length %u exceeds max %u",
....@@ -347,11 +543,80 @@
347543 		error(1, errno, "setsockopt path mtu");
348544 }
349545 
546
+static void set_tx_timestamping(int fd)
547
+{
548
+	int val = SOF_TIMESTAMPING_OPT_CMSG | SOF_TIMESTAMPING_OPT_ID |
549
+			SOF_TIMESTAMPING_OPT_TSONLY;
550
+
551
+	if (cfg_tx_ts == SOF_TIMESTAMPING_TX_SOFTWARE)
552
+		val |= SOF_TIMESTAMPING_SOFTWARE;
553
+	else
554
+		val |= SOF_TIMESTAMPING_RAW_HARDWARE;
555
+
556
+	if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &val, sizeof(val)))
557
+		error(1, errno, "setsockopt tx timestamping");
558
+}
559
+
560
+static void print_audit_report(unsigned long num_msgs, unsigned long num_sends)
561
+{
562
+	unsigned long tdelta;
563
+
564
+	tdelta = tend - tstart;
565
+	if (!tdelta)
566
+		return;
567
+
568
+	fprintf(stderr, "Summary over %lu.%03lu seconds...\n",
569
+			tdelta / 1000, tdelta % 1000);
570
+	fprintf(stderr,
571
+		"sum %s tx: %6lu MB/s %10lu calls (%lu/s) %10lu msgs (%lu/s)\n",
572
+		cfg_tcp ? "tcp" : "udp",
573
+		((num_msgs * cfg_payload_len) >> 10) / tdelta,
574
+		num_sends, num_sends * 1000 / tdelta,
575
+		num_msgs, num_msgs * 1000 / tdelta);
576
+
577
+	if (cfg_tx_tstamp) {
578
+		if (stat_tx_ts_errors)
579
+			error(1, 0,
580
+			      "Expected clean TX Timestamps: %9lu msgs received %6lu errors",
581
+			      stat_tx_ts, stat_tx_ts_errors);
582
+		if (stat_tx_ts != num_sends)
583
+			error(1, 0,
584
+			      "Unexpected number of TX Timestamps: %9lu expected %9lu received",
585
+			      num_sends, stat_tx_ts);
586
+		fprintf(stderr,
587
+			"Tx Timestamps: %19lu received %17lu errors\n",
588
+			stat_tx_ts, stat_tx_ts_errors);
589
+	}
590
+
591
+	if (cfg_zerocopy) {
592
+		if (stat_zcopies != num_sends)
593
+			error(1, 0, "Unexpected number of Zerocopy completions: %9lu expected %9lu received",
594
+			      num_sends, stat_zcopies);
595
+		fprintf(stderr,
596
+			"Zerocopy acks: %19lu\n",
597
+			stat_zcopies);
598
+	}
599
+}
600
+
601
+static void print_report(unsigned long num_msgs, unsigned long num_sends)
602
+{
603
+	fprintf(stderr,
604
+		"%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n",
605
+		cfg_tcp ? "tcp" : "udp",
606
+		(num_msgs * cfg_payload_len) >> 20,
607
+		num_sends, num_msgs);
608
+
609
+	if (cfg_audit) {
610
+		total_num_msgs += num_msgs;
611
+		total_num_sends += num_sends;
612
+	}
613
+}
614
+
350615 int main(int argc, char **argv)
351616 {
352617 	unsigned long num_msgs, num_sends;
353618 	unsigned long tnow, treport, tstop;
354
-	int fd, i, val;
619
+	int fd, i, val, ret;
355620 
356621 	parse_opts(argc, argv);
357622 
....@@ -371,8 +636,16 @@
371636 
372637 	if (cfg_zerocopy) {
373638 		val = 1;
374
-		if (setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val)))
639
+
640
+		ret = setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,
641
+				 &val, sizeof(val));
642
+		if (ret) {
643
+			if (errno == ENOPROTOOPT || errno == ENOTSUPP) {
644
+				fprintf(stderr, "SO_ZEROCOPY not supported");
645
+				exit(KSFT_SKIP);
646
+			}
375647 			error(1, errno, "setsockopt zerocopy");
648
+		}
376649 	}
377650 
378651 	if (cfg_connected &&
....@@ -382,8 +655,13 @@
382655 	if (cfg_segment)
383656 		set_pmtu_discover(fd, cfg_family == PF_INET);
384657 
658
+	if (cfg_tx_tstamp)
659
+		set_tx_timestamping(fd);
660
+
385661 	num_msgs = num_sends = 0;
386662 	tnow = gettimeofday_ms();
663
+	tstart = tnow;
664
+	tend = tnow;
387665 	tstop = tnow + cfg_runtime_ms;
388666 	treport = tnow + 1000;
389667 
....@@ -398,17 +676,15 @@
398676 		else
399677 			num_sends += send_udp(fd, buf[i]);
400678 		num_msgs++;
679
+		if ((cfg_zerocopy && ((num_msgs & 0xF) == 0)) || cfg_tx_tstamp)
680
+			flush_errqueue(fd, cfg_poll);
401681 
402
-		if (cfg_zerocopy && ((num_msgs & 0xF) == 0))
403
-			flush_zerocopy(fd);
682
+		if (cfg_msg_nr && num_msgs >= cfg_msg_nr)
683
+			break;
404684 
405685 		tnow = gettimeofday_ms();
406
-		if (tnow > treport) {
407
-			fprintf(stderr,
408
-				"%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n",
409
-				cfg_tcp ? "tcp" : "udp",
410
-				(num_msgs * cfg_payload_len) >> 20,
411
-				num_sends, num_msgs);
686
+		if (tnow >= treport) {
687
+			print_report(num_msgs, num_sends);
412688 			num_msgs = num_sends = 0;
413689 			treport = tnow + 1000;
414690 		}
....@@ -419,8 +695,18 @@
419695 
420696 	} while (!interrupted && (cfg_runtime_ms == -1 || tnow < tstop));
421697 
698
+	if (cfg_zerocopy || cfg_tx_tstamp)
699
+		flush_errqueue(fd, true);
700
+
422701 	if (close(fd))
423702 		error(1, errno, "close");
424703 
704
+	if (cfg_audit) {
705
+		tend = tnow;
706
+		total_num_msgs += num_msgs;
707
+		total_num_sends += num_sends;
708
+		print_audit_report(total_num_msgs, total_num_sends);
709
+	}
710
+
425711 	return 0;
426712 }