~hc/RK356X_SDK_RELEASE.git

..	..	@@ -10,7 +10,6 @@
10	10	#include <unistd.h>
11	11	#include <string.h>
12	12	#include <errno.h>
13		-#include <sys/ioctl.h>
14	13	#include <stdbool.h>
15	14	#include <signal.h>
16	15	#include <fcntl.h>
..	..	@@ -28,6 +27,7 @@
28	27	#include <linux/sock_diag.h>
29	28	#include <linux/bpf.h>
30	29	#include <linux/if_link.h>
	30	+#include <linux/tls.h>
31	31	#include <assert.h>
32	32	#include <libgen.h>
33	33
..	..	@@ -43,11 +43,18 @@
43	43	int running;
44	44	static void running_handler(int a);
45	45
	46	+#ifndef TCP_ULP
	47	+# define TCP_ULP 31
	48	+#endif
	49	+#ifndef SOL_TLS
	50	+# define SOL_TLS 282
	51	+#endif
	52	+
46	53	/* randomly selected ports for testing on lo */
47	54	#define S1_PORT 10000
48	55	#define S2_PORT 10001
49	56
50		-#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
	57	+#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
51	58	#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
52	59	#define CG_PATH "/sockmap"
53	60
..	..	@@ -56,44 +63,156 @@
56	63	int test_cnt;
57	64	int passed;
58	65	int failed;
59		-int map_fd[8];
60		-struct bpf_map *maps[8];
	66	+int map_fd[9];
	67	+struct bpf_map *maps[9];
61	68	int prog_fd[11];
62	69
63	70	int txmsg_pass;
64		-int txmsg_noisy;
65	71	int txmsg_redir;
66		-int txmsg_redir_noisy;
67	72	int txmsg_drop;
68	73	int txmsg_apply;
69	74	int txmsg_cork;
70	75	int txmsg_start;
71	76	int txmsg_end;
	77	+int txmsg_start_push;
	78	+int txmsg_end_push;
	79	+int txmsg_start_pop;
	80	+int txmsg_pop;
72	81	int txmsg_ingress;
73		-int txmsg_skb;
	82	+int txmsg_redir_skb;
	83	+int txmsg_ktls_skb;
	84	+int txmsg_ktls_skb_drop;
	85	+int txmsg_ktls_skb_redir;
	86	+int ktls;
	87	+int peek_flag;
	88	+int skb_use_parser;
	89	+int txmsg_omit_skb_parser;
74	90
75	91	static const struct option long_options[] = {
76	92	{"help", no_argument, NULL, 'h' },
77	93	{"cgroup", required_argument, NULL, 'c' },
78	94	{"rate", required_argument, NULL, 'r' },
79		- {"verbose", no_argument, NULL, 'v' },
	95	+ {"verbose", optional_argument, NULL, 'v' },
80	96	{"iov_count", required_argument, NULL, 'i' },
81	97	{"length", required_argument, NULL, 'l' },
82	98	{"test", required_argument, NULL, 't' },
83	99	{"data_test", no_argument, NULL, 'd' },
84	100	{"txmsg", no_argument, &txmsg_pass, 1 },
85		- {"txmsg_noisy", no_argument, &txmsg_noisy, 1 },
86	101	{"txmsg_redir", no_argument, &txmsg_redir, 1 },
87		- {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1},
88	102	{"txmsg_drop", no_argument, &txmsg_drop, 1 },
89	103	{"txmsg_apply", required_argument, NULL, 'a'},
90	104	{"txmsg_cork", required_argument, NULL, 'k'},
91	105	{"txmsg_start", required_argument, NULL, 's'},
92	106	{"txmsg_end", required_argument, NULL, 'e'},
	107	+ {"txmsg_start_push", required_argument, NULL, 'p'},
	108	+ {"txmsg_end_push", required_argument, NULL, 'q'},
	109	+ {"txmsg_start_pop", required_argument, NULL, 'w'},
	110	+ {"txmsg_pop", required_argument, NULL, 'x'},
93	111	{"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
94		- {"txmsg_skb", no_argument, &txmsg_skb, 1 },
	112	+ {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
	113	+ {"ktls", no_argument, &ktls, 1 },
	114	+ {"peek", no_argument, &peek_flag, 1 },
	115	+ {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
	116	+ {"whitelist", required_argument, NULL, 'n' },
	117	+ {"blacklist", required_argument, NULL, 'b' },
95	118	{0, 0, NULL, 0 }
96	119	};
	120	+
	121	+struct test_env {
	122	+ const char *type;
	123	+ const char *subtest;
	124	+ const char *prepend;
	125	+
	126	+ int test_num;
	127	+ int subtest_num;
	128	+
	129	+ int succ_cnt;
	130	+ int fail_cnt;
	131	+ int fail_last;
	132	+};
	133	+
	134	+struct test_env env;
	135	+
	136	+struct sockmap_options {
	137	+ int verbose;
	138	+ bool base;
	139	+ bool sendpage;
	140	+ bool data_test;
	141	+ bool drop_expected;
	142	+ int iov_count;
	143	+ int iov_length;
	144	+ int rate;
	145	+ char *map;
	146	+ char *whitelist;
	147	+ char *blacklist;
	148	+ char *prepend;
	149	+};
	150	+
	151	+struct _test {
	152	+ char *title;
	153	+ void (tester)(int cg_fd, struct sockmap_options opt);
	154	+};
	155	+
	156	+static void test_start(void)
	157	+{
	158	+ env.subtest_num++;
	159	+}
	160	+
	161	+static void test_fail(void)
	162	+{
	163	+ env.fail_cnt++;
	164	+}
	165	+
	166	+static void test_pass(void)
	167	+{
	168	+ env.succ_cnt++;
	169	+}
	170	+
	171	+static void test_reset(void)
	172	+{
	173	+ txmsg_start = txmsg_end = 0;
	174	+ txmsg_start_pop = txmsg_pop = 0;
	175	+ txmsg_start_push = txmsg_end_push = 0;
	176	+ txmsg_pass = txmsg_drop = txmsg_redir = 0;
	177	+ txmsg_apply = txmsg_cork = 0;
	178	+ txmsg_ingress = txmsg_redir_skb = 0;
	179	+ txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
	180	+ txmsg_omit_skb_parser = 0;
	181	+ skb_use_parser = 0;
	182	+}
	183	+
	184	+static int test_start_subtest(const struct _test t, struct sockmap_options o)
	185	+{
	186	+ env.type = o->map;
	187	+ env.subtest = t->title;
	188	+ env.prepend = o->prepend;
	189	+ env.test_num++;
	190	+ env.subtest_num = 0;
	191	+ env.fail_last = env.fail_cnt;
	192	+ test_reset();
	193	+ return 0;
	194	+}
	195	+
	196	+static void test_end_subtest(void)
	197	+{
	198	+ int error = env.fail_cnt - env.fail_last;
	199	+ int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
	200	+
	201	+ if (!error)
	202	+ test_pass();
	203	+
	204	+ fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
	205	+ env.test_num, env.subtest_num,
	206	+ !type ? "sockmap" : "sockhash",
	207	+ env.prepend ? : "",
	208	+ env.subtest, error ? "FAIL" : "OK");
	209	+}
	210	+
	211	+static void test_print_results(void)
	212	+{
	213	+ fprintf(stdout, "Pass: %d Fail: %d\n",
	214	+ env.succ_cnt, env.fail_cnt);
	215	+}
97	216
98	217	static void usage(char *argv[])
99	218	{
..	..	@@ -112,6 +231,71 @@
112	231	printf("\n");
113	232	}
114	233
	234	+char *sock_to_string(int s)
	235	+{
	236	+ if (s == c1)
	237	+ return "client1";
	238	+ else if (s == c2)
	239	+ return "client2";
	240	+ else if (s == s1)
	241	+ return "server1";
	242	+ else if (s == s2)
	243	+ return "server2";
	244	+ else if (s == p1)
	245	+ return "peer1";
	246	+ else if (s == p2)
	247	+ return "peer2";
	248	+ else
	249	+ return "unknown";
	250	+}
	251	+
	252	+static int sockmap_init_ktls(int verbose, int s)
	253	+{
	254	+ struct tls12_crypto_info_aes_gcm_128 tls_tx = {
	255	+ .info = {
	256	+ .version = TLS_1_2_VERSION,
	257	+ .cipher_type = TLS_CIPHER_AES_GCM_128,
	258	+ },
	259	+ };
	260	+ struct tls12_crypto_info_aes_gcm_128 tls_rx = {
	261	+ .info = {
	262	+ .version = TLS_1_2_VERSION,
	263	+ .cipher_type = TLS_CIPHER_AES_GCM_128,
	264	+ },
	265	+ };
	266	+ int so_buf = 6553500;
	267	+ int err;
	268	+
	269	+ err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
	270	+ if (err) {
	271	+ fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
	272	+ return -EINVAL;
	273	+ }
	274	+ err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
	275	+ if (err) {
	276	+ fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
	277	+ return -EINVAL;
	278	+ }
	279	+ err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
	280	+ if (err) {
	281	+ fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
	282	+ return -EINVAL;
	283	+ }
	284	+ err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
	285	+ if (err) {
	286	+ fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
	287	+ return -EINVAL;
	288	+ }
	289	+ err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
	290	+ if (err) {
	291	+ fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
	292	+ return -EINVAL;
	293	+ }
	294	+
	295	+ if (verbose)
	296	+ fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
	297	+ return 0;
	298	+}
115	299	static int sockmap_init_sockets(int verbose)
116	300	{
117	301	int i, err, one = 1;
..	..	@@ -156,14 +340,14 @@
156	340	addr.sin_port = htons(S1_PORT);
157	341	err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
158	342	if (err < 0) {
159		- perror("bind s1 failed()\n");
	343	+ perror("bind s1 failed()");
160	344	return errno;
161	345	}
162	346
163	347	addr.sin_port = htons(S2_PORT);
164	348	err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
165	349	if (err < 0) {
166		- perror("bind s2 failed()\n");
	350	+ perror("bind s2 failed()");
167	351	return errno;
168	352	}
169	353
..	..	@@ -171,14 +355,14 @@
171	355	addr.sin_port = htons(S1_PORT);
172	356	err = listen(s1, 32);
173	357	if (err < 0) {
174		- perror("listen s1 failed()\n");
	358	+ perror("listen s1 failed()");
175	359	return errno;
176	360	}
177	361
178	362	addr.sin_port = htons(S2_PORT);
179	363	err = listen(s2, 32);
180	364	if (err < 0) {
181		- perror("listen s1 failed()\n");
	365	+ perror("listen s1 failed()");
182	366	return errno;
183	367	}
184	368
..	..	@@ -186,14 +370,14 @@
186	370	addr.sin_port = htons(S1_PORT);
187	371	err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
188	372	if (err < 0 && errno != EINPROGRESS) {
189		- perror("connect c1 failed()\n");
	373	+ perror("connect c1 failed()");
190	374	return errno;
191	375	}
192	376
193	377	addr.sin_port = htons(S2_PORT);
194	378	err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
195	379	if (err < 0 && errno != EINPROGRESS) {
196		- perror("connect c2 failed()\n");
	380	+ perror("connect c2 failed()");
197	381	return errno;
198	382	} else if (err < 0) {
199	383	err = 0;
..	..	@@ -202,17 +386,17 @@
202	386	/* Accept Connecrtions */
203	387	p1 = accept(s1, NULL, NULL);
204	388	if (p1 < 0) {
205		- perror("accept s1 failed()\n");
	389	+ perror("accept s1 failed()");
206	390	return errno;
207	391	}
208	392
209	393	p2 = accept(s2, NULL, NULL);
210	394	if (p2 < 0) {
211		- perror("accept s1 failed()\n");
	395	+ perror("accept s1 failed()");
212	396	return errno;
213	397	}
214	398
215		- if (verbose) {
	399	+ if (verbose > 1) {
216	400	printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
217	401	printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
218	402	c1, s1, c2, s2);
..	..	@@ -227,17 +411,6 @@
227	411	struct timespec end;
228	412	};
229	413
230		-struct sockmap_options {
231		- int verbose;
232		- bool base;
233		- bool sendpage;
234		- bool data_test;
235		- bool drop_expected;
236		- int iov_count;
237		- int iov_length;
238		- int rate;
239		-};
240		-
241	414	static int msg_loop_sendpage(int fd, int iov_length, int cnt,
242	415	struct msg_stats *s,
243	416	struct sockmap_options *opt)
..	..	@@ -247,7 +420,7 @@
247	420	FILE *file;
248	421	int i, fp;
249	422
250		- file = fopen(".sendpage_tst.tmp", "w+");
	423	+ file = tmpfile();
251	424	if (!file) {
252	425	perror("create file for sendpage");
253	426	return 1;
..	..	@@ -256,25 +429,24 @@
256	429	fwrite(&k, sizeof(char), 1, file);
257	430	fflush(file);
258	431	fseek(file, 0, SEEK_SET);
259		- fclose(file);
260	432
261		- fp = open(".sendpage_tst.tmp", O_RDONLY);
262		- if (fp < 0) {
263		- perror("reopen file for sendpage");
264		- return 1;
265		- }
	433	+ fp = fileno(file);
266	434
267	435	clock_gettime(CLOCK_MONOTONIC, &s->start);
268	436	for (i = 0; i < cnt; i++) {
269		- int sent = sendfile(fd, fp, NULL, iov_length);
	437	+ int sent;
	438	+
	439	+ errno = 0;
	440	+ sent = sendfile(fd, fp, NULL, iov_length);
270	441
271	442	if (!drop && sent < 0) {
272		- perror("send loop error:");
273		- close(fp);
	443	+ perror("sendpage loop error");
	444	+ fclose(file);
274	445	return sent;
275	446	} else if (drop && sent >= 0) {
276		- printf("sendpage loop error expected: %i\n", sent);
277		- close(fp);
	447	+ printf("sendpage loop error expected: %i errno %i\n",
	448	+ sent, errno);
	449	+ fclose(file);
278	450	return -EIO;
279	451	}
280	452
..	..	@@ -282,37 +454,44 @@
282	454	s->bytes_sent += sent;
283	455	}
284	456	clock_gettime(CLOCK_MONOTONIC, &s->end);
285		- close(fp);
	457	+ fclose(file);
286	458	return 0;
287	459	}
288	460
289		-static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
290		- struct msg_stats *s, bool tx,
291		- struct sockmap_options *opt)
	461	+static void msg_free_iov(struct msghdr *msg)
292	462	{
293		- struct msghdr msg = {0};
294		- int err, i, flags = MSG_NOSIGNAL;
	463	+ int i;
	464	+
	465	+ for (i = 0; i < msg->msg_iovlen; i++)
	466	+ free(msg->msg_iov[i].iov_base);
	467	+ free(msg->msg_iov);
	468	+ msg->msg_iov = NULL;
	469	+ msg->msg_iovlen = 0;
	470	+}
	471	+
	472	+static int msg_alloc_iov(struct msghdr *msg,
	473	+ int iov_count, int iov_length,
	474	+ bool data, bool xmit)
	475	+{
	476	+ unsigned char k = 0;
295	477	struct iovec *iov;
296		- unsigned char k;
297		- bool data_test = opt->data_test;
298		- bool drop = opt->drop_expected;
	478	+ int i;
299	479
300	480	iov = calloc(iov_count, sizeof(struct iovec));
301	481	if (!iov)
302	482	return errno;
303	483
304		- k = 0;
305	484	for (i = 0; i < iov_count; i++) {
306	485	unsigned char *d = calloc(iov_length, sizeof(char));
307	486
308	487	if (!d) {
309	488	fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
310		- goto out_errno;
	489	+ goto unwind_iov;
311	490	}
312	491	iov[i].iov_base = d;
313	492	iov[i].iov_len = iov_length;
314	493
315		- if (data_test && tx) {
	494	+ if (data && xmit) {
316	495	int j;
317	496
318	497	for (j = 0; j < iov_length; j++)
..	..	@@ -320,20 +499,88 @@
320	499	}
321	500	}
322	501
323		- msg.msg_iov = iov;
324		- msg.msg_iovlen = iov_count;
325		- k = 0;
	502	+ msg->msg_iov = iov;
	503	+ msg->msg_iovlen = iov_count;
	504	+
	505	+ return 0;
	506	+unwind_iov:
	507	+ for (i--; i >= 0 ; i--)
	508	+ free(msg->msg_iov[i].iov_base);
	509	+ return -ENOMEM;
	510	+}
	511	+
	512	+static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
	513	+{
	514	+ int i, j = 0, bytes_cnt = 0;
	515	+ unsigned char k = 0;
	516	+
	517	+ for (i = 0; i < msg->msg_iovlen; i++) {
	518	+ unsigned char *d = msg->msg_iov[i].iov_base;
	519	+
	520	+ /* Special case test for skb ingress + ktls */
	521	+ if (i == 0 && txmsg_ktls_skb) {
	522	+ if (msg->msg_iov[i].iov_len < 4)
	523	+ return -EIO;
	524	+ if (memcmp(d, "PASS", 4) != 0) {
	525	+ fprintf(stderr,
	526	+ "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
	527	+ i, 0, d[0], d[1], d[2], d[3]);
	528	+ return -EIO;
	529	+ }
	530	+ j = 4; /* advance index past PASS header */
	531	+ }
	532	+
	533	+ for (; j < msg->msg_iov[i].iov_len && size; j++) {
	534	+ if (d[j] != k++) {
	535	+ fprintf(stderr,
	536	+ "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
	537	+ i, j, d[j], k - 1, d[j+1], k);
	538	+ return -EIO;
	539	+ }
	540	+ bytes_cnt++;
	541	+ if (bytes_cnt == chunk_sz) {
	542	+ k = 0;
	543	+ bytes_cnt = 0;
	544	+ }
	545	+ size--;
	546	+ }
	547	+ }
	548	+ return 0;
	549	+}
	550	+
	551	+static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
	552	+ struct msg_stats *s, bool tx,
	553	+ struct sockmap_options *opt)
	554	+{
	555	+ struct msghdr msg = {0}, msg_peek = {0};
	556	+ int err, i, flags = MSG_NOSIGNAL;
	557	+ bool drop = opt->drop_expected;
	558	+ bool data = opt->data_test;
	559	+
	560	+ err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx);
	561	+ if (err)
	562	+ goto out_errno;
	563	+ if (peek_flag) {
	564	+ err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
	565	+ if (err)
	566	+ goto out_errno;
	567	+ }
326	568
327	569	if (tx) {
328	570	clock_gettime(CLOCK_MONOTONIC, &s->start);
329	571	for (i = 0; i < cnt; i++) {
330		- int sent = sendmsg(fd, &msg, flags);
	572	+ int sent;
	573	+
	574	+ errno = 0;
	575	+ sent = sendmsg(fd, &msg, flags);
331	576
332	577	if (!drop && sent < 0) {
333		- perror("send loop error:");
	578	+ perror("sendmsg loop error");
334	579	goto out_errno;
335	580	} else if (drop && sent >= 0) {
336		- printf("send loop error expected: %i\n", sent);
	581	+ fprintf(stderr,
	582	+ "sendmsg loop error expected: %i errno %i\n",
	583	+ sent, errno);
337	584	errno = -EIO;
338	585	goto out_errno;
339	586	}
..	..	@@ -342,30 +589,38 @@
342	589	}
343	590	clock_gettime(CLOCK_MONOTONIC, &s->end);
344	591	} else {
345		- int slct, recv, max_fd = fd;
	592	+ int slct, recvp = 0, recv, max_fd = fd;
	593	+ float total_bytes, txmsg_pop_total;
346	594	int fd_flags = O_NONBLOCK;
347	595	struct timeval timeout;
348		- float total_bytes;
349		- int bytes_cnt = 0;
350		- int chunk_sz;
351	596	fd_set w;
352	597
353		- if (opt->sendpage)
354		- chunk_sz = iov_length * cnt;
355		- else
356		- chunk_sz = iov_length * iov_count;
357		-
358	598	fcntl(fd, fd_flags);
	599	+ /* Account for pop bytes noting each iteration of apply will
	600	+ * call msg_pop_data helper so we need to account for this
	601	+ * by calculating the number of apply iterations. Note user
	602	+ * of the tool can create cases where no data is sent by
	603	+ * manipulating pop/push/pull/etc. For example txmsg_apply 1
	604	+ * with txmsg_pop 1 will try to apply 1B at a time but each
	605	+ * iteration will then pop 1B so no data will ever be sent.
	606	+ * This is really only useful for testing edge cases in code
	607	+ * paths.
	608	+ */
359	609	total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
	610	+ if (txmsg_apply)
	611	+ txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
	612	+ else
	613	+ txmsg_pop_total = txmsg_pop * cnt;
	614	+ total_bytes -= txmsg_pop_total;
360	615	err = clock_gettime(CLOCK_MONOTONIC, &s->start);
361	616	if (err < 0)
362		- perror("recv start time: ");
	617	+ perror("recv start time");
363	618	while (s->bytes_recvd < total_bytes) {
364	619	if (txmsg_cork) {
365	620	timeout.tv_sec = 0;
366	621	timeout.tv_usec = 300000;
367	622	} else {
368		- timeout.tv_sec = 1;
	623	+ timeout.tv_sec = 3;
369	624	timeout.tv_usec = 0;
370	625	}
371	626
..	..	@@ -380,44 +635,53 @@
380	635	goto out_errno;
381	636	} else if (!slct) {
382	637	if (opt->verbose)
383		- fprintf(stderr, "unexpected timeout\n");
	638	+ fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
384	639	errno = -EIO;
385	640	clock_gettime(CLOCK_MONOTONIC, &s->end);
386	641	goto out_errno;
	642	+ }
	643	+
	644	+ errno = 0;
	645	+ if (peek_flag) {
	646	+ flags \|= MSG_PEEK;
	647	+ recvp = recvmsg(fd, &msg_peek, flags);
	648	+ if (recvp < 0) {
	649	+ if (errno != EWOULDBLOCK) {
	650	+ clock_gettime(CLOCK_MONOTONIC, &s->end);
	651	+ goto out_errno;
	652	+ }
	653	+ }
	654	+ flags = 0;
387	655	}
388	656
389	657	recv = recvmsg(fd, &msg, flags);
390	658	if (recv < 0) {
391	659	if (errno != EWOULDBLOCK) {
392	660	clock_gettime(CLOCK_MONOTONIC, &s->end);
393		- perror("recv failed()\n");
	661	+ perror("recv failed()");
394	662	goto out_errno;
395	663	}
396	664	}
397	665
398	666	s->bytes_recvd += recv;
399	667
400		- if (data_test) {
401		- int j;
	668	+ if (data) {
	669	+ int chunk_sz = opt->sendpage ?
	670	+ iov_length * cnt :
	671	+ iov_length * iov_count;
402	672
403		- for (i = 0; i < msg.msg_iovlen; i++) {
404		- unsigned char *d = iov[i].iov_base;
405		-
406		- for (j = 0;
407		- j < iov[i].iov_len && recv; j++) {
408		- if (d[j] != k++) {
409		- errno = -EIO;
410		- fprintf(stderr,
411		- "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
412		- i, j, d[j], k - 1, d[j+1], k);
413		- goto out_errno;
414		- }
415		- bytes_cnt++;
416		- if (bytes_cnt == chunk_sz) {
417		- k = 0;
418		- bytes_cnt = 0;
419		- }
420		- recv--;
	673	+ errno = msg_verify_data(&msg, recv, chunk_sz);
	674	+ if (errno) {
	675	+ perror("data verify msg failed");
	676	+ goto out_errno;
	677	+ }
	678	+ if (recvp) {
	679	+ errno = msg_verify_data(&msg_peek,
	680	+ recvp,
	681	+ chunk_sz);
	682	+ if (errno) {
	683	+ perror("data verify msg_peek failed");
	684	+ goto out_errno;
421	685	}
422	686	}
423	687	}
..	..	@@ -425,14 +689,12 @@
425	689	clock_gettime(CLOCK_MONOTONIC, &s->end);
426	690	}
427	691
428		- for (i = 0; i < iov_count; i++)
429		- free(iov[i].iov_base);
430		- free(iov);
431		- return 0;
	692	+ msg_free_iov(&msg);
	693	+ msg_free_iov(&msg_peek);
	694	+ return err;
432	695	out_errno:
433		- for (i = 0; i < iov_count; i++)
434		- free(iov[i].iov_base);
435		- free(iov);
	696	+ msg_free_iov(&msg);
	697	+ msg_free_iov(&msg_peek);
436	698	return errno;
437	699	}
438	700
..	..	@@ -465,35 +727,53 @@
465	727	else
466	728	rx_fd = p2;
467	729
	730	+ if (ktls) {
	731	+ /* Redirecting into non-TLS socket which sends into a TLS
	732	+ * socket is not a valid test. So in this case lets not
	733	+ * enable kTLS but still run the test.
	734	+ */
	735	+ if (!txmsg_redir \|\| (txmsg_redir && txmsg_ingress)) {
	736	+ err = sockmap_init_ktls(opt->verbose, rx_fd);
	737	+ if (err)
	738	+ return err;
	739	+ }
	740	+ err = sockmap_init_ktls(opt->verbose, c1);
	741	+ if (err)
	742	+ return err;
	743	+ }
	744	+
468	745	rxpid = fork();
469	746	if (rxpid == 0) {
470		- if (opt->drop_expected)
471		- exit(0);
	747	+ iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
	748	+ if (opt->drop_expected \|\| txmsg_ktls_skb_drop)
	749	+ _exit(0);
	750	+
	751	+ if (!iov_buf) /* zero bytes sent case */
	752	+ _exit(0);
472	753
473	754	if (opt->sendpage)
474	755	iov_count = 1;
475	756	err = msg_loop(rx_fd, iov_count, iov_buf,
476	757	cnt, &s, false, opt);
477		- if (err && opt->verbose)
	758	+ if (opt->verbose > 1)
478	759	fprintf(stderr,
479	760	"msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
480	761	iov_count, iov_buf, cnt, err);
481		- shutdown(p2, SHUT_RDWR);
482		- shutdown(p1, SHUT_RDWR);
483	762	if (s.end.tv_sec - s.start.tv_sec) {
484	763	sent_Bps = sentBps(s);
485	764	recvd_Bps = recvdBps(s);
486	765	}
487		- if (opt->verbose)
	766	+ if (opt->verbose > 1)
488	767	fprintf(stdout,
489		- "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s\n",
	768	+ "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
490	769	s.bytes_sent, sent_Bps, sent_Bps/giga,
491		- s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
	770	+ s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
	771	+ peek_flag ? "(peek_msg)" : "");
492	772	if (err && txmsg_cork)
493	773	err = 0;
494	774	exit(err ? 1 : 0);
495	775	} else if (rxpid == -1) {
496		- perror("msg_loop_rx: ");
	776	+ perror("msg_loop_rx");
497	777	return errno;
498	778	}
499	779
..	..	@@ -509,19 +789,18 @@
509	789	fprintf(stderr,
510	790	"msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
511	791	iov_count, iov_buf, cnt, err);
512		- shutdown(c1, SHUT_RDWR);
513	792	if (s.end.tv_sec - s.start.tv_sec) {
514	793	sent_Bps = sentBps(s);
515	794	recvd_Bps = recvdBps(s);
516	795	}
517		- if (opt->verbose)
	796	+ if (opt->verbose > 1)
518	797	fprintf(stdout,
519	798	"tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
520	799	s.bytes_sent, sent_Bps, sent_Bps/giga,
521	800	s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
522	801	exit(err ? 1 : 0);
523	802	} else if (txpid == -1) {
524		- perror("msg_loop_tx: ");
	803	+ perror("msg_loop_tx");
525	804	return errno;
526	805	}
527	806
..	..	@@ -530,14 +809,14 @@
530	809	if (WIFEXITED(rx_status)) {
531	810	err = WEXITSTATUS(rx_status);
532	811	if (err) {
533		- fprintf(stderr, "rx thread exited with err %d. ", err);
	812	+ fprintf(stderr, "rx thread exited with err %d.\n", err);
534	813	goto out;
535	814	}
536	815	}
537	816	if (WIFEXITED(tx_status)) {
538	817	err = WEXITSTATUS(tx_status);
539	818	if (err)
540		- fprintf(stderr, "tx thread exited with err %d. ", err);
	819	+ fprintf(stderr, "tx thread exited with err %d.\n", err);
541	820	}
542	821	out:
543	822	return err;
..	..	@@ -555,7 +834,7 @@
555	834	/* Ping/Pong data from client to server */
556	835	sc = send(c1, buf, sizeof(buf), 0);
557	836	if (sc < 0) {
558		- perror("send failed()\n");
	837	+ perror("send failed()");
559	838	return sc;
560	839	}
561	840
..	..	@@ -588,7 +867,7 @@
588	867	rc = recv(i, buf, sizeof(buf), 0);
589	868	if (rc < 0) {
590	869	if (errno != EWOULDBLOCK) {
591		- perror("recv failed()\n");
	870	+ perror("recv failed()");
592	871	return rc;
593	872	}
594	873	}
..	..	@@ -600,7 +879,7 @@
600	879
601	880	sc = send(i, buf, rc, 0);
602	881	if (sc < 0) {
603		- perror("send failed()\n");
	882	+ perror("send failed()");
604	883	return sc;
605	884	}
606	885	}
..	..	@@ -619,6 +898,7 @@
619	898	}
620	899
621	900	enum {
	901	+ SELFTESTS,
622	902	PING_PONG,
623	903	SENDMSG,
624	904	BASE,
..	..	@@ -635,13 +915,15 @@
635	915	goto run;
636	916
637	917	/* Attach programs to sockmap */
638		- err = bpf_prog_attach(prog_fd[0], map_fd[0],
639		- BPF_SK_SKB_STREAM_PARSER, 0);
640		- if (err) {
641		- fprintf(stderr,
642		- "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
643		- prog_fd[0], map_fd[0], err, strerror(errno));
644		- return err;
	918	+ if (!txmsg_omit_skb_parser) {
	919	+ err = bpf_prog_attach(prog_fd[0], map_fd[0],
	920	+ BPF_SK_SKB_STREAM_PARSER, 0);
	921	+ if (err) {
	922	+ fprintf(stderr,
	923	+ "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
	924	+ prog_fd[0], map_fd[0], err, strerror(errno));
	925	+ return err;
	926	+ }
645	927	}
646	928
647	929	err = bpf_prog_attach(prog_fd[1], map_fd[0],
..	..	@@ -652,8 +934,30 @@
652	934	return err;
653	935	}
654	936
	937	+ /* Attach programs to TLS sockmap */
	938	+ if (txmsg_ktls_skb) {
	939	+ if (!txmsg_omit_skb_parser) {
	940	+ err = bpf_prog_attach(prog_fd[0], map_fd[8],
	941	+ BPF_SK_SKB_STREAM_PARSER, 0);
	942	+ if (err) {
	943	+ fprintf(stderr,
	944	+ "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
	945	+ prog_fd[0], map_fd[8], err, strerror(errno));
	946	+ return err;
	947	+ }
	948	+ }
	949	+
	950	+ err = bpf_prog_attach(prog_fd[2], map_fd[8],
	951	+ BPF_SK_SKB_STREAM_VERDICT, 0);
	952	+ if (err) {
	953	+ fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
	954	+ err, strerror(errno));
	955	+ return err;
	956	+ }
	957	+ }
	958	+
655	959	/* Attach to cgroups */
656		- err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
	960	+ err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
657	961	if (err) {
658	962	fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
659	963	err, strerror(errno));
..	..	@@ -669,19 +973,14 @@
669	973
670	974	/* Attach txmsg program to sockmap */
671	975	if (txmsg_pass)
672		- tx_prog_fd = prog_fd[3];
673		- else if (txmsg_noisy)
674	976	tx_prog_fd = prog_fd[4];
675	977	else if (txmsg_redir)
676	978	tx_prog_fd = prog_fd[5];
677		- else if (txmsg_redir_noisy)
678		- tx_prog_fd = prog_fd[6];
679		- else if (txmsg_drop)
680		- tx_prog_fd = prog_fd[9];
681		- /* apply and cork must be last */
682	979	else if (txmsg_apply)
683		- tx_prog_fd = prog_fd[7];
	980	+ tx_prog_fd = prog_fd[6];
684	981	else if (txmsg_cork)
	982	+ tx_prog_fd = prog_fd[7];
	983	+ else if (txmsg_drop)
685	984	tx_prog_fd = prog_fd[8];
686	985	else
687	986	tx_prog_fd = 0;
..	..	@@ -706,7 +1005,7 @@
706	1005	goto out;
707	1006	}
708	1007
709		- if (txmsg_redir \|\| txmsg_redir_noisy)
	1008	+ if (txmsg_redir)
710	1009	redir_fd = c2;
711	1010	else
712	1011	redir_fd = c1;
..	..	@@ -764,6 +1063,63 @@
764	1063	}
765	1064	}
766	1065
	1066	+ if (txmsg_start_push) {
	1067	+ i = 2;
	1068	+ err = bpf_map_update_elem(map_fd[5],
	1069	+ &i, &txmsg_start_push, BPF_ANY);
	1070	+ if (err) {
	1071	+ fprintf(stderr,
	1072	+ "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
	1073	+ err, strerror(errno));
	1074	+ goto out;
	1075	+ }
	1076	+ }
	1077	+
	1078	+ if (txmsg_end_push) {
	1079	+ i = 3;
	1080	+ err = bpf_map_update_elem(map_fd[5],
	1081	+ &i, &txmsg_end_push, BPF_ANY);
	1082	+ if (err) {
	1083	+ fprintf(stderr,
	1084	+ "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
	1085	+ txmsg_end_push, i, err, strerror(errno));
	1086	+ goto out;
	1087	+ }
	1088	+ }
	1089	+
	1090	+ if (txmsg_start_pop) {
	1091	+ i = 4;
	1092	+ err = bpf_map_update_elem(map_fd[5],
	1093	+ &i, &txmsg_start_pop, BPF_ANY);
	1094	+ if (err) {
	1095	+ fprintf(stderr,
	1096	+ "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
	1097	+ txmsg_start_pop, i, err, strerror(errno));
	1098	+ goto out;
	1099	+ }
	1100	+ } else {
	1101	+ i = 4;
	1102	+ bpf_map_update_elem(map_fd[5],
	1103	+ &i, &txmsg_start_pop, BPF_ANY);
	1104	+ }
	1105	+
	1106	+ if (txmsg_pop) {
	1107	+ i = 5;
	1108	+ err = bpf_map_update_elem(map_fd[5],
	1109	+ &i, &txmsg_pop, BPF_ANY);
	1110	+ if (err) {
	1111	+ fprintf(stderr,
	1112	+ "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
	1113	+ txmsg_pop, i, err, strerror(errno));
	1114	+ goto out;
	1115	+ }
	1116	+ } else {
	1117	+ i = 5;
	1118	+ bpf_map_update_elem(map_fd[5],
	1119	+ &i, &txmsg_pop, BPF_ANY);
	1120	+
	1121	+ }
	1122	+
767	1123	if (txmsg_ingress) {
768	1124	int in = BPF_F_INGRESS;
769	1125
..	..	@@ -797,7 +1153,35 @@
797	1153	}
798	1154	}
799	1155
800		- if (txmsg_skb) {
	1156	+ if (txmsg_ktls_skb) {
	1157	+ int ingress = BPF_F_INGRESS;
	1158	+
	1159	+ i = 0;
	1160	+ err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
	1161	+ if (err) {
	1162	+ fprintf(stderr,
	1163	+ "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
	1164	+ err, strerror(errno));
	1165	+ }
	1166	+
	1167	+ if (txmsg_ktls_skb_redir) {
	1168	+ i = 1;
	1169	+ err = bpf_map_update_elem(map_fd[7],
	1170	+ &i, &ingress, BPF_ANY);
	1171	+ if (err) {
	1172	+ fprintf(stderr,
	1173	+ "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
	1174	+ err, strerror(errno));
	1175	+ }
	1176	+ }
	1177	+
	1178	+ if (txmsg_ktls_skb_drop) {
	1179	+ i = 1;
	1180	+ err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
	1181	+ }
	1182	+ }
	1183	+
	1184	+ if (txmsg_redir_skb) {
801	1185	int skb_fd = (test == SENDMSG \|\| test == SENDPAGE) ?
802	1186	p2 : p1;
803	1187	int ingress = BPF_F_INGRESS;
..	..	@@ -812,14 +1196,18 @@
812	1196	}
813	1197
814	1198	i = 3;
815		- err = bpf_map_update_elem(map_fd[0],
816		- &i, &skb_fd, BPF_ANY);
	1199	+ err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
817	1200	if (err) {
818	1201	fprintf(stderr,
819	1202	"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
820	1203	err, strerror(errno));
821	1204	}
822	1205	}
	1206	+ }
	1207	+
	1208	+ if (skb_use_parser) {
	1209	+ i = 2;
	1210	+ err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
823	1211	}
824	1212
825	1213	if (txmsg_drop)
..	..	@@ -847,9 +1235,12 @@
847	1235	fprintf(stderr, "unknown test\n");
848	1236	out:
849	1237	/* Detatch and zero all the maps */
850		- bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS);
	1238	+ bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
851	1239	bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
852	1240	bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
	1241	+ bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
	1242	+ bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
	1243	+
853	1244	if (tx_prog_fd >= 0)
854	1245	bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
855	1246
..	..	@@ -882,6 +1273,16 @@
882	1273	return "unknown";
883	1274	}
884	1275
	1276	+static void append_str(char dst, const char src, size_t dst_cap)
	1277	+{
	1278	+ size_t avail = dst_cap - strlen(dst);
	1279	+
	1280	+ if (avail <= 1) /* just zero byte could be written */
	1281	+ return;
	1282	+
	1283	+ strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
	1284	+}
	1285	+
885	1286	#define OPTSTRING 60
886	1287	static void test_options(char *options)
887	1288	{
..	..	@@ -890,35 +1291,42 @@
890	1291	memset(options, 0, OPTSTRING);
891	1292
892	1293	if (txmsg_pass)
893		- strncat(options, "pass,", OPTSTRING);
894		- if (txmsg_noisy)
895		- strncat(options, "pass_noisy,", OPTSTRING);
	1294	+ append_str(options, "pass,", OPTSTRING);
896	1295	if (txmsg_redir)
897		- strncat(options, "redir,", OPTSTRING);
898		- if (txmsg_redir_noisy)
899		- strncat(options, "redir_noisy,", OPTSTRING);
	1296	+ append_str(options, "redir,", OPTSTRING);
900	1297	if (txmsg_drop)
901		- strncat(options, "drop,", OPTSTRING);
	1298	+ append_str(options, "drop,", OPTSTRING);
902	1299	if (txmsg_apply) {
903	1300	snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
904		- strncat(options, tstr, OPTSTRING);
	1301	+ append_str(options, tstr, OPTSTRING);
905	1302	}
906	1303	if (txmsg_cork) {
907	1304	snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
908		- strncat(options, tstr, OPTSTRING);
	1305	+ append_str(options, tstr, OPTSTRING);
909	1306	}
910	1307	if (txmsg_start) {
911	1308	snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
912		- strncat(options, tstr, OPTSTRING);
	1309	+ append_str(options, tstr, OPTSTRING);
913	1310	}
914	1311	if (txmsg_end) {
915	1312	snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
916		- strncat(options, tstr, OPTSTRING);
	1313	+ append_str(options, tstr, OPTSTRING);
	1314	+ }
	1315	+ if (txmsg_start_pop) {
	1316	+ snprintf(tstr, OPTSTRING, "pop (%d,%d),",
	1317	+ txmsg_start_pop, txmsg_start_pop + txmsg_pop);
	1318	+ append_str(options, tstr, OPTSTRING);
917	1319	}
918	1320	if (txmsg_ingress)
919		- strncat(options, "ingress,", OPTSTRING);
920		- if (txmsg_skb)
921		- strncat(options, "skb,", OPTSTRING);
	1321	+ append_str(options, "ingress,", OPTSTRING);
	1322	+ if (txmsg_redir_skb)
	1323	+ append_str(options, "redir_skb,", OPTSTRING);
	1324	+ if (txmsg_ktls_skb)
	1325	+ append_str(options, "ktls_skb,", OPTSTRING);
	1326	+ if (ktls)
	1327	+ append_str(options, "ktls,", OPTSTRING);
	1328	+ if (peek_flag)
	1329	+ append_str(options, "peek,", OPTSTRING);
922	1330	}
923	1331
924	1332	static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
..	..	@@ -938,341 +1346,344 @@
938	1346
939	1347	test_options(options);
940	1348
941		- fprintf(stdout,
942		- "[TEST %i]: (%i, %i, %i, %s, %s): ",
943		- test_cnt, opt->rate, opt->iov_count, opt->iov_length,
944		- test_to_str(test), options);
945		- fflush(stdout);
	1349	+ if (opt->verbose) {
	1350	+ fprintf(stdout,
	1351	+ " [TEST %i]: (%i, %i, %i, %s, %s): ",
	1352	+ test_cnt, opt->rate, opt->iov_count, opt->iov_length,
	1353	+ test_to_str(test), options);
	1354	+ fflush(stdout);
	1355	+ }
946	1356	err = run_options(opt, cgrp, test);
947		- fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED");
	1357	+ if (opt->verbose)
	1358	+ fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
948	1359	test_cnt++;
949	1360	!err ? passed++ : failed++;
950	1361	free(options);
951	1362	return err;
952	1363	}
953	1364
954		-static int test_exec(int cgrp, struct sockmap_options *opt)
	1365	+static void test_exec(int cgrp, struct sockmap_options *opt)
955	1366	{
956		- int err = __test_exec(cgrp, SENDMSG, opt);
	1367	+ int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
	1368	+ int err;
957	1369
958		- if (err)
959		- goto out;
960		-
961		- err = __test_exec(cgrp, SENDPAGE, opt);
962		-out:
963		- return err;
964		-}
965		-
966		-static int test_loop(int cgrp)
967		-{
968		- struct sockmap_options opt;
969		-
970		- int err, i, l, r;
971		-
972		- opt.verbose = 0;
973		- opt.base = false;
974		- opt.sendpage = false;
975		- opt.data_test = false;
976		- opt.drop_expected = false;
977		- opt.iov_count = 0;
978		- opt.iov_length = 0;
979		- opt.rate = 0;
980		-
981		- r = 1;
982		- for (i = 1; i < 100; i += 33) {
983		- for (l = 1; l < 100; l += 33) {
984		- opt.rate = r;
985		- opt.iov_count = i;
986		- opt.iov_length = l;
987		- err = test_exec(cgrp, &opt);
988		- if (err)
989		- goto out;
990		- }
	1370	+ if (type == 0) {
	1371	+ test_start();
	1372	+ err = __test_exec(cgrp, SENDMSG, opt);
	1373	+ if (err)
	1374	+ test_fail();
	1375	+ } else {
	1376	+ test_start();
	1377	+ err = __test_exec(cgrp, SENDPAGE, opt);
	1378	+ if (err)
	1379	+ test_fail();
991	1380	}
992		- sched_yield();
993		-out:
994		- return err;
995	1381	}
996	1382
997		-static int test_txmsg(int cgrp)
	1383	+static void test_send_one(struct sockmap_options *opt, int cgrp)
998	1384	{
999		- int err;
1000		-
1001		- txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1002		- txmsg_apply = txmsg_cork = 0;
1003		- txmsg_ingress = txmsg_skb = 0;
1004		-
1005		- txmsg_pass = 1;
1006		- err = test_loop(cgrp);
1007		- txmsg_pass = 0;
1008		- if (err)
1009		- goto out;
1010		-
1011		- txmsg_redir = 1;
1012		- err = test_loop(cgrp);
1013		- txmsg_redir = 0;
1014		- if (err)
1015		- goto out;
1016		-
1017		- txmsg_drop = 1;
1018		- err = test_loop(cgrp);
1019		- txmsg_drop = 0;
1020		- if (err)
1021		- goto out;
1022		-
1023		- txmsg_redir = 1;
1024		- txmsg_ingress = 1;
1025		- err = test_loop(cgrp);
1026		- txmsg_redir = 0;
1027		- txmsg_ingress = 0;
1028		- if (err)
1029		- goto out;
1030		-out:
1031		- txmsg_pass = 0;
1032		- txmsg_redir = 0;
1033		- txmsg_drop = 0;
1034		- return err;
1035		-}
1036		-
1037		-static int test_send(struct sockmap_options *opt, int cgrp)
1038		-{
1039		- int err;
1040		-
1041	1385	opt->iov_length = 1;
1042	1386	opt->iov_count = 1;
1043	1387	opt->rate = 1;
1044		- err = test_exec(cgrp, opt);
1045		- if (err)
1046		- goto out;
	1388	+ test_exec(cgrp, opt);
1047	1389
1048	1390	opt->iov_length = 1;
1049	1391	opt->iov_count = 1024;
1050	1392	opt->rate = 1;
1051		- err = test_exec(cgrp, opt);
1052		- if (err)
1053		- goto out;
	1393	+ test_exec(cgrp, opt);
1054	1394
1055	1395	opt->iov_length = 1024;
1056	1396	opt->iov_count = 1;
1057	1397	opt->rate = 1;
1058		- err = test_exec(cgrp, opt);
1059		- if (err)
1060		- goto out;
	1398	+ test_exec(cgrp, opt);
1061	1399
1062		- opt->iov_length = 1;
	1400	+}
	1401	+
	1402	+static void test_send_many(struct sockmap_options *opt, int cgrp)
	1403	+{
	1404	+ opt->iov_length = 3;
1063	1405	opt->iov_count = 1;
1064	1406	opt->rate = 512;
1065		- err = test_exec(cgrp, opt);
1066		- if (err)
1067		- goto out;
1068		-
1069		- opt->iov_length = 256;
1070		- opt->iov_count = 1024;
1071		- opt->rate = 2;
1072		- err = test_exec(cgrp, opt);
1073		- if (err)
1074		- goto out;
	1407	+ test_exec(cgrp, opt);
1075	1408
1076	1409	opt->rate = 100;
1077	1410	opt->iov_count = 1;
1078	1411	opt->iov_length = 5;
1079		- err = test_exec(cgrp, opt);
1080		- if (err)
1081		- goto out;
1082		-out:
1083		- sched_yield();
1084		- return err;
	1412	+ test_exec(cgrp, opt);
1085	1413	}
1086	1414
1087		-static int test_mixed(int cgrp)
	1415	+static void test_send_large(struct sockmap_options *opt, int cgrp)
1088	1416	{
1089		- struct sockmap_options opt = {0};
1090		- int err;
	1417	+ opt->iov_length = 256;
	1418	+ opt->iov_count = 1024;
	1419	+ opt->rate = 2;
	1420	+ test_exec(cgrp, opt);
	1421	+}
1091	1422
1092		- txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1093		- txmsg_apply = txmsg_cork = 0;
1094		- txmsg_start = txmsg_end = 0;
	1423	+static void test_send(struct sockmap_options *opt, int cgrp)
	1424	+{
	1425	+ test_send_one(opt, cgrp);
	1426	+ test_send_many(opt, cgrp);
	1427	+ test_send_large(opt, cgrp);
	1428	+ sched_yield();
	1429	+}
	1430	+
	1431	+static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
	1432	+{
1095	1433	/* Test small and large iov_count values with pass/redir/apply/cork */
1096	1434	txmsg_pass = 1;
1097		- txmsg_redir = 0;
1098		- txmsg_apply = 1;
1099		- txmsg_cork = 0;
1100		- err = test_send(&opt, cgrp);
1101		- if (err)
1102		- goto out;
1103		-
1104		- txmsg_pass = 1;
1105		- txmsg_redir = 0;
1106		- txmsg_apply = 0;
1107		- txmsg_cork = 1;
1108		- err = test_send(&opt, cgrp);
1109		- if (err)
1110		- goto out;
1111		-
1112		- txmsg_pass = 1;
1113		- txmsg_redir = 0;
1114		- txmsg_apply = 1;
1115		- txmsg_cork = 1;
1116		- err = test_send(&opt, cgrp);
1117		- if (err)
1118		- goto out;
1119		-
1120		- txmsg_pass = 1;
1121		- txmsg_redir = 0;
1122		- txmsg_apply = 1024;
1123		- txmsg_cork = 0;
1124		- err = test_send(&opt, cgrp);
1125		- if (err)
1126		- goto out;
1127		-
1128		- txmsg_pass = 1;
1129		- txmsg_redir = 0;
1130		- txmsg_apply = 0;
1131		- txmsg_cork = 1024;
1132		- err = test_send(&opt, cgrp);
1133		- if (err)
1134		- goto out;
1135		-
1136		- txmsg_pass = 1;
1137		- txmsg_redir = 0;
1138		- txmsg_apply = 1024;
1139		- txmsg_cork = 1024;
1140		- err = test_send(&opt, cgrp);
1141		- if (err)
1142		- goto out;
1143		-
1144		- txmsg_pass = 1;
1145		- txmsg_redir = 0;
1146		- txmsg_cork = 4096;
1147		- txmsg_apply = 4096;
1148		- err = test_send(&opt, cgrp);
1149		- if (err)
1150		- goto out;
1151		-
1152		- txmsg_pass = 0;
1153		- txmsg_redir = 1;
1154		- txmsg_apply = 1;
1155		- txmsg_cork = 0;
1156		- err = test_send(&opt, cgrp);
1157		- if (err)
1158		- goto out;
1159		-
1160		- txmsg_pass = 0;
1161		- txmsg_redir = 1;
1162		- txmsg_apply = 0;
1163		- txmsg_cork = 1;
1164		- err = test_send(&opt, cgrp);
1165		- if (err)
1166		- goto out;
1167		-
1168		- txmsg_pass = 0;
1169		- txmsg_redir = 1;
1170		- txmsg_apply = 1024;
1171		- txmsg_cork = 0;
1172		- err = test_send(&opt, cgrp);
1173		- if (err)
1174		- goto out;
1175		-
1176		- txmsg_pass = 0;
1177		- txmsg_redir = 1;
1178		- txmsg_apply = 0;
1179		- txmsg_cork = 1024;
1180		- err = test_send(&opt, cgrp);
1181		- if (err)
1182		- goto out;
1183		-
1184		- txmsg_pass = 0;
1185		- txmsg_redir = 1;
1186		- txmsg_apply = 1024;
1187		- txmsg_cork = 1024;
1188		- err = test_send(&opt, cgrp);
1189		- if (err)
1190		- goto out;
1191		-
1192		- txmsg_pass = 0;
1193		- txmsg_redir = 1;
1194		- txmsg_cork = 4096;
1195		- txmsg_apply = 4096;
1196		- err = test_send(&opt, cgrp);
1197		- if (err)
1198		- goto out;
1199		-out:
1200		- return err;
	1435	+ test_send(opt, cgrp);
1201	1436	}
1202	1437
1203		-static int test_start_end(int cgrp)
	1438	+static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1204	1439	{
1205		- struct sockmap_options opt = {0};
1206		- int err, i;
	1440	+ txmsg_redir = 1;
	1441	+ test_send(opt, cgrp);
	1442	+}
1207	1443
1208		- /* Test basic start/end with lots of iov_count and iov_lengths */
	1444	+static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
	1445	+{
	1446	+ txmsg_drop = 1;
	1447	+ test_send(opt, cgrp);
	1448	+}
	1449	+
	1450	+static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
	1451	+{
	1452	+ txmsg_pass = txmsg_drop = 0;
	1453	+ txmsg_ingress = txmsg_redir = 1;
	1454	+ test_send(opt, cgrp);
	1455	+}
	1456	+
	1457	+static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
	1458	+{
	1459	+ bool data = opt->data_test;
	1460	+ int k = ktls;
	1461	+
	1462	+ opt->data_test = true;
	1463	+ ktls = 1;
	1464	+
	1465	+ txmsg_pass = txmsg_drop = 0;
	1466	+ txmsg_ingress = txmsg_redir = 0;
	1467	+ txmsg_ktls_skb = 1;
	1468	+ txmsg_pass = 1;
	1469	+
	1470	+ /* Using data verification so ensure iov layout is
	1471	+ * expected from test receiver side. e.g. has enough
	1472	+ * bytes to write test code.
	1473	+ */
	1474	+ opt->iov_length = 100;
	1475	+ opt->iov_count = 1;
	1476	+ opt->rate = 1;
	1477	+ test_exec(cgrp, opt);
	1478	+
	1479	+ txmsg_ktls_skb_drop = 1;
	1480	+ test_exec(cgrp, opt);
	1481	+
	1482	+ txmsg_ktls_skb_drop = 0;
	1483	+ txmsg_ktls_skb_redir = 1;
	1484	+ test_exec(cgrp, opt);
	1485	+ txmsg_ktls_skb_redir = 0;
	1486	+
	1487	+ /* Tests that omit skb_parser */
	1488	+ txmsg_omit_skb_parser = 1;
	1489	+ ktls = 0;
	1490	+ txmsg_ktls_skb = 0;
	1491	+ test_exec(cgrp, opt);
	1492	+
	1493	+ txmsg_ktls_skb_drop = 1;
	1494	+ test_exec(cgrp, opt);
	1495	+ txmsg_ktls_skb_drop = 0;
	1496	+
	1497	+ txmsg_ktls_skb_redir = 1;
	1498	+ test_exec(cgrp, opt);
	1499	+
	1500	+ ktls = 1;
	1501	+ test_exec(cgrp, opt);
	1502	+ txmsg_omit_skb_parser = 0;
	1503	+
	1504	+ opt->data_test = data;
	1505	+ ktls = k;
	1506	+}
	1507	+
	1508	+/* Test cork with hung data. This tests poor usage patterns where
	1509	+ * cork can leave data on the ring if user program is buggy and
	1510	+ * doesn't flush them somehow. They do take some time however
	1511	+ * because they wait for a timeout. Test pass, redir and cork with
	1512	+ * apply logic. Use cork size of 4097 with send_large to avoid
	1513	+ * aligning cork size with send size.
	1514	+ */
	1515	+static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
	1516	+{
	1517	+ txmsg_pass = 1;
	1518	+ txmsg_redir = 0;
	1519	+ txmsg_cork = 4097;
	1520	+ txmsg_apply = 4097;
	1521	+ test_send_large(opt, cgrp);
	1522	+
	1523	+ txmsg_pass = 0;
	1524	+ txmsg_redir = 1;
	1525	+ txmsg_apply = 0;
	1526	+ txmsg_cork = 4097;
	1527	+ test_send_large(opt, cgrp);
	1528	+
	1529	+ txmsg_pass = 0;
	1530	+ txmsg_redir = 1;
	1531	+ txmsg_apply = 4097;
	1532	+ txmsg_cork = 4097;
	1533	+ test_send_large(opt, cgrp);
	1534	+}
	1535	+
	1536	+static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
	1537	+{
	1538	+ /* Test basic start/end */
1209	1539	txmsg_start = 1;
1210	1540	txmsg_end = 2;
1211		- err = test_txmsg(cgrp);
1212		- if (err)
1213		- goto out;
	1541	+ test_send(opt, cgrp);
1214	1542
1215		- /* Test start/end with cork */
1216		- opt.rate = 16;
1217		- opt.iov_count = 1;
1218		- opt.iov_length = 100;
1219		- txmsg_cork = 1600;
	1543	+ /* Test >4k pull */
	1544	+ txmsg_start = 4096;
	1545	+ txmsg_end = 9182;
	1546	+ test_send_large(opt, cgrp);
1220	1547
1221		- for (i = 99; i <= 1600; i += 500) {
1222		- txmsg_start = 0;
1223		- txmsg_end = i;
1224		- err = test_exec(cgrp, &opt);
1225		- if (err)
1226		- goto out;
1227		- }
	1548	+ /* Test pull + redirect */
	1549	+ txmsg_redir = 0;
	1550	+ txmsg_start = 1;
	1551	+ txmsg_end = 2;
	1552	+ test_send(opt, cgrp);
1228	1553
1229		- /* Test start/end with cork but pull data in middle */
1230		- for (i = 199; i <= 1600; i += 500) {
1231		- txmsg_start = 100;
1232		- txmsg_end = i;
1233		- err = test_exec(cgrp, &opt);
1234		- if (err)
1235		- goto out;
1236		- }
	1554	+ /* Test pull + cork */
	1555	+ txmsg_redir = 0;
	1556	+ txmsg_cork = 512;
	1557	+ txmsg_start = 1;
	1558	+ txmsg_end = 2;
	1559	+ test_send_many(opt, cgrp);
1237	1560
1238		- /* Test start/end with cork pulling last sg entry */
1239		- txmsg_start = 1500;
1240		- txmsg_end = 1600;
1241		- err = test_exec(cgrp, &opt);
1242		- if (err)
1243		- goto out;
	1561	+ /* Test pull + cork + redirect */
	1562	+ txmsg_redir = 1;
	1563	+ txmsg_cork = 512;
	1564	+ txmsg_start = 1;
	1565	+ txmsg_end = 2;
	1566	+ test_send_many(opt, cgrp);
	1567	+}
1244	1568
1245		- /* Test start/end pull of single byte in last page */
1246		- txmsg_start = 1111;
1247		- txmsg_end = 1112;
1248		- err = test_exec(cgrp, &opt);
1249		- if (err)
1250		- goto out;
	1569	+static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
	1570	+{
	1571	+ /* Test basic pop */
	1572	+ txmsg_start_pop = 1;
	1573	+ txmsg_pop = 2;
	1574	+ test_send_many(opt, cgrp);
1251	1575
1252		- /* Test start/end with end < start */
1253		- txmsg_start = 1111;
1254		- txmsg_end = 0;
1255		- err = test_exec(cgrp, &opt);
1256		- if (err)
1257		- goto out;
	1576	+ /* Test pop with >4k */
	1577	+ txmsg_start_pop = 4096;
	1578	+ txmsg_pop = 4096;
	1579	+ test_send_large(opt, cgrp);
1258	1580
1259		- /* Test start/end with end > data */
1260		- txmsg_start = 0;
1261		- txmsg_end = 1601;
1262		- err = test_exec(cgrp, &opt);
1263		- if (err)
1264		- goto out;
	1581	+ /* Test pop + redirect */
	1582	+ txmsg_redir = 1;
	1583	+ txmsg_start_pop = 1;
	1584	+ txmsg_pop = 2;
	1585	+ test_send_many(opt, cgrp);
1265	1586
1266		- /* Test start/end with start > data */
1267		- txmsg_start = 1601;
1268		- txmsg_end = 1600;
1269		- err = test_exec(cgrp, &opt);
	1587	+ /* Test pop + cork */
	1588	+ txmsg_redir = 0;
	1589	+ txmsg_cork = 512;
	1590	+ txmsg_start_pop = 1;
	1591	+ txmsg_pop = 2;
	1592	+ test_send_many(opt, cgrp);
1270	1593
1271		-out:
1272		- txmsg_start = 0;
1273		- txmsg_end = 0;
1274		- sched_yield();
1275		- return err;
	1594	+ /* Test pop + redirect + cork */
	1595	+ txmsg_redir = 1;
	1596	+ txmsg_cork = 4;
	1597	+ txmsg_start_pop = 1;
	1598	+ txmsg_pop = 2;
	1599	+ test_send_many(opt, cgrp);
	1600	+}
	1601	+
	1602	+static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
	1603	+{
	1604	+ /* Test basic push */
	1605	+ txmsg_start_push = 1;
	1606	+ txmsg_end_push = 1;
	1607	+ test_send(opt, cgrp);
	1608	+
	1609	+ /* Test push 4kB >4k */
	1610	+ txmsg_start_push = 4096;
	1611	+ txmsg_end_push = 4096;
	1612	+ test_send_large(opt, cgrp);
	1613	+
	1614	+ /* Test push + redirect */
	1615	+ txmsg_redir = 1;
	1616	+ txmsg_start_push = 1;
	1617	+ txmsg_end_push = 2;
	1618	+ test_send_many(opt, cgrp);
	1619	+
	1620	+ /* Test push + cork */
	1621	+ txmsg_redir = 0;
	1622	+ txmsg_cork = 512;
	1623	+ txmsg_start_push = 1;
	1624	+ txmsg_end_push = 2;
	1625	+ test_send_many(opt, cgrp);
	1626	+}
	1627	+
	1628	+static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
	1629	+{
	1630	+ txmsg_start_push = 1;
	1631	+ txmsg_end_push = 10;
	1632	+ txmsg_start_pop = 5;
	1633	+ txmsg_pop = 4;
	1634	+ test_send_large(opt, cgrp);
	1635	+}
	1636	+
	1637	+static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
	1638	+{
	1639	+ txmsg_pass = 1;
	1640	+ txmsg_redir = 0;
	1641	+ txmsg_apply = 1;
	1642	+ txmsg_cork = 0;
	1643	+ test_send_one(opt, cgrp);
	1644	+
	1645	+ txmsg_pass = 0;
	1646	+ txmsg_redir = 1;
	1647	+ txmsg_apply = 1;
	1648	+ txmsg_cork = 0;
	1649	+ test_send_one(opt, cgrp);
	1650	+
	1651	+ txmsg_pass = 1;
	1652	+ txmsg_redir = 0;
	1653	+ txmsg_apply = 1024;
	1654	+ txmsg_cork = 0;
	1655	+ test_send_large(opt, cgrp);
	1656	+
	1657	+ txmsg_pass = 0;
	1658	+ txmsg_redir = 1;
	1659	+ txmsg_apply = 1024;
	1660	+ txmsg_cork = 0;
	1661	+ test_send_large(opt, cgrp);
	1662	+}
	1663	+
	1664	+static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
	1665	+{
	1666	+ txmsg_pass = 1;
	1667	+ txmsg_redir = 0;
	1668	+ txmsg_apply = 0;
	1669	+ txmsg_cork = 1;
	1670	+ test_send(opt, cgrp);
	1671	+
	1672	+ txmsg_pass = 1;
	1673	+ txmsg_redir = 0;
	1674	+ txmsg_apply = 1;
	1675	+ txmsg_cork = 1;
	1676	+ test_send(opt, cgrp);
	1677	+}
	1678	+
	1679	+static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
	1680	+{
	1681	+ txmsg_pass = 1;
	1682	+ skb_use_parser = 512;
	1683	+ opt->iov_length = 256;
	1684	+ opt->iov_count = 1;
	1685	+ opt->rate = 2;
	1686	+ test_exec(cgrp, opt);
1276	1687	}
1277	1688
1278	1689	char *map_names[] = {
..	..	@@ -1281,13 +1692,15 @@
1281	1692	"sock_map_redir",
1282	1693	"sock_apply_bytes",
1283	1694	"sock_cork_bytes",
1284		- "sock_pull_bytes",
	1695	+ "sock_bytes",
1285	1696	"sock_redir_flags",
1286	1697	"sock_skb_opts",
	1698	+ "tls_sock_map",
1287	1699	};
1288	1700
1289	1701	int prog_attach_type[] = {
1290	1702	BPF_SK_SKB_STREAM_PARSER,
	1703	+ BPF_SK_SKB_STREAM_VERDICT,
1291	1704	BPF_SK_SKB_STREAM_VERDICT,
1292	1705	BPF_CGROUP_SOCK_OPS,
1293	1706	BPF_SK_MSG_VERDICT,
..	..	@@ -1300,6 +1713,7 @@
1300	1713	};
1301	1714
1302	1715	int prog_type[] = {
	1716	+ BPF_PROG_TYPE_SK_SKB,
1303	1717	BPF_PROG_TYPE_SK_SKB,
1304	1718	BPF_PROG_TYPE_SK_SKB,
1305	1719	BPF_PROG_TYPE_SOCK_OPS,
..	..	@@ -1357,67 +1771,118 @@
1357	1771	return 0;
1358	1772	}
1359	1773
1360		-static int __test_suite(char *bpf_file)
1361		-{
1362		- int cg_fd, err;
	1774	+struct _test test[] = {
	1775	+ {"txmsg test passthrough", test_txmsg_pass},
	1776	+ {"txmsg test redirect", test_txmsg_redir},
	1777	+ {"txmsg test drop", test_txmsg_drop},
	1778	+ {"txmsg test ingress redirect", test_txmsg_ingress_redir},
	1779	+ {"txmsg test skb", test_txmsg_skb},
	1780	+ {"txmsg test apply", test_txmsg_apply},
	1781	+ {"txmsg test cork", test_txmsg_cork},
	1782	+ {"txmsg test hanging corks", test_txmsg_cork_hangs},
	1783	+ {"txmsg test push_data", test_txmsg_push},
	1784	+ {"txmsg test pull-data", test_txmsg_pull},
	1785	+ {"txmsg test pop-data", test_txmsg_pop},
	1786	+ {"txmsg test push/pop data", test_txmsg_push_pop},
	1787	+ {"txmsg text ingress parser", test_txmsg_ingress_parser},
	1788	+};
1363	1789
1364		- err = populate_progs(bpf_file);
	1790	+static int check_whitelist(struct _test t, struct sockmap_options opt)
	1791	+{
	1792	+ char entry, ptr;
	1793	+
	1794	+ if (!opt->whitelist)
	1795	+ return 0;
	1796	+ ptr = strdup(opt->whitelist);
	1797	+ if (!ptr)
	1798	+ return -ENOMEM;
	1799	+ entry = strtok(ptr, ",");
	1800	+ while (entry) {
	1801	+ if ((opt->prepend && strstr(opt->prepend, entry) != 0) \|\|
	1802	+ strstr(opt->map, entry) != 0 \|\|
	1803	+ strstr(t->title, entry) != 0)
	1804	+ return 0;
	1805	+ entry = strtok(NULL, ",");
	1806	+ }
	1807	+ return -EINVAL;
	1808	+}
	1809	+
	1810	+static int check_blacklist(struct _test t, struct sockmap_options opt)
	1811	+{
	1812	+ char entry, ptr;
	1813	+
	1814	+ if (!opt->blacklist)
	1815	+ return -EINVAL;
	1816	+ ptr = strdup(opt->blacklist);
	1817	+ if (!ptr)
	1818	+ return -ENOMEM;
	1819	+ entry = strtok(ptr, ",");
	1820	+ while (entry) {
	1821	+ if ((opt->prepend && strstr(opt->prepend, entry) != 0) \|\|
	1822	+ strstr(opt->map, entry) != 0 \|\|
	1823	+ strstr(t->title, entry) != 0)
	1824	+ return 0;
	1825	+ entry = strtok(NULL, ",");
	1826	+ }
	1827	+ return -EINVAL;
	1828	+}
	1829	+
	1830	+static int __test_selftests(int cg_fd, struct sockmap_options *opt)
	1831	+{
	1832	+ int i, err;
	1833	+
	1834	+ err = populate_progs(opt->map);
1365	1835	if (err < 0) {
1366	1836	fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1367	1837	return err;
1368	1838	}
1369	1839
1370		- if (setup_cgroup_environment()) {
1371		- fprintf(stderr, "ERROR: cgroup env failed\n");
1372		- return -EINVAL;
	1840	+ /* Tests basic commands and APIs */
	1841	+ for (i = 0; i < sizeof(test)/sizeof(struct _test); i++) {
	1842	+ struct _test t = test[i];
	1843	+
	1844	+ if (check_whitelist(&t, opt) != 0)
	1845	+ continue;
	1846	+ if (check_blacklist(&t, opt) == 0)
	1847	+ continue;
	1848	+
	1849	+ test_start_subtest(&t, opt);
	1850	+ t.tester(cg_fd, opt);
	1851	+ test_end_subtest();
1373	1852	}
1374	1853
1375		- cg_fd = create_and_get_cgroup(CG_PATH);
1376		- if (cg_fd < 0) {
1377		- fprintf(stderr,
1378		- "ERROR: (%i) open cg path failed: %s\n",
1379		- cg_fd, optarg);
1380		- return cg_fd;
1381		- }
1382		-
1383		- if (join_cgroup(CG_PATH)) {
1384		- fprintf(stderr, "ERROR: failed to join cgroup\n");
1385		- return -EINVAL;
1386		- }
1387		-
1388		- /* Tests basic commands and APIs with range of iov values */
1389		- txmsg_start = txmsg_end = 0;
1390		- err = test_txmsg(cg_fd);
1391		- if (err)
1392		- goto out;
1393		-
1394		- /* Tests interesting combinations of APIs used together */
1395		- err = test_mixed(cg_fd);
1396		- if (err)
1397		- goto out;
1398		-
1399		- /* Tests pull_data API using start/end API */
1400		- err = test_start_end(cg_fd);
1401		- if (err)
1402		- goto out;
1403		-
1404		-out:
1405		- printf("Summary: %i PASSED %i FAILED\n", passed, failed);
1406		- cleanup_cgroup_environment();
1407		- close(cg_fd);
1408	1854	return err;
1409	1855	}
1410	1856
1411		-static int test_suite(void)
	1857	+static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1412	1858	{
1413		- int err;
	1859	+ opt->map = BPF_SOCKMAP_FILENAME;
	1860	+ __test_selftests(cg_fd, opt);
	1861	+}
1414	1862
1415		- err = __test_suite(BPF_SOCKMAP_FILENAME);
1416		- if (err)
1417		- goto out;
1418		- err = __test_suite(BPF_SOCKHASH_FILENAME);
1419		-out:
1420		- return err;
	1863	+static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
	1864	+{
	1865	+ opt->map = BPF_SOCKHASH_FILENAME;
	1866	+ __test_selftests(cg_fd, opt);
	1867	+}
	1868	+
	1869	+static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
	1870	+{
	1871	+ opt->map = BPF_SOCKHASH_FILENAME;
	1872	+ opt->prepend = "ktls";
	1873	+ ktls = 1;
	1874	+ __test_selftests(cg_fd, opt);
	1875	+ ktls = 0;
	1876	+}
	1877	+
	1878	+static int test_selftest(int cg_fd, struct sockmap_options *opt)
	1879	+{
	1880	+
	1881	+ test_selftests_sockmap(cg_fd, opt);
	1882	+ test_selftests_sockhash(cg_fd, opt);
	1883	+ test_selftests_ktls(cg_fd, opt);
	1884	+ test_print_results();
	1885	+ return 0;
1421	1886	}
1422	1887
1423	1888	int main(int argc, char **argv)
..	..	@@ -1426,12 +1891,10 @@
1426	1891	struct sockmap_options options = {0};
1427	1892	int opt, longindex, err, cg_fd = 0;
1428	1893	char *bpf_file = BPF_SOCKMAP_FILENAME;
1429		- int test = PING_PONG;
	1894	+ int test = SELFTESTS;
	1895	+ bool cg_created = 0;
1430	1896
1431		- if (argc < 2)
1432		- return test_suite();
1433		-
1434		- while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:",
	1897	+ while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1435	1898	long_options, &longindex)) != -1) {
1436	1899	switch (opt) {
1437	1900	case 's':
..	..	@@ -1439,6 +1902,18 @@
1439	1902	break;
1440	1903	case 'e':
1441	1904	txmsg_end = atoi(optarg);
	1905	+ break;
	1906	+ case 'p':
	1907	+ txmsg_start_push = atoi(optarg);
	1908	+ break;
	1909	+ case 'q':
	1910	+ txmsg_end_push = atoi(optarg);
	1911	+ break;
	1912	+ case 'w':
	1913	+ txmsg_start_pop = atoi(optarg);
	1914	+ break;
	1915	+ case 'x':
	1916	+ txmsg_pop = atoi(optarg);
1442	1917	break;
1443	1918	case 'a':
1444	1919	txmsg_apply = atoi(optarg);
..	..	@@ -1460,6 +1935,8 @@
1460	1935	break;
1461	1936	case 'v':
1462	1937	options.verbose = 1;
	1938	+ if (optarg)
	1939	+ options.verbose = atoi(optarg);
1463	1940	break;
1464	1941	case 'i':
1465	1942	iov_count = atoi(optarg);
..	..	@@ -1486,6 +1963,15 @@
1486	1963	return -1;
1487	1964	}
1488	1965	break;
	1966	+ case 'n':
	1967	+ options.whitelist = strdup(optarg);
	1968	+ if (!options.whitelist)
	1969	+ return -ENOMEM;
	1970	+ break;
	1971	+ case 'b':
	1972	+ options.blacklist = strdup(optarg);
	1973	+ if (!options.blacklist)
	1974	+ return -ENOMEM;
1489	1975	case 0:
1490	1976	break;
1491	1977	case 'h':
..	..	@@ -1496,9 +1982,15 @@
1496	1982	}
1497	1983
1498	1984	if (!cg_fd) {
1499		- fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n",
1500		- argv[0]);
1501		- return -1;
	1985	+ cg_fd = cgroup_setup_and_join(CG_PATH);
	1986	+ if (cg_fd < 0)
	1987	+ return cg_fd;
	1988	+ cg_created = 1;
	1989	+ }
	1990	+
	1991	+ if (test == SELFTESTS) {
	1992	+ err = test_selftest(cg_fd, &options);
	1993	+ goto out;
1502	1994	}
1503	1995
1504	1996	err = populate_progs(bpf_file);
..	..	@@ -1517,6 +2009,13 @@
1517	2009	options.rate = rate;
1518	2010
1519	2011	err = run_options(&options, cg_fd, test);
	2012	+out:
	2013	+ if (options.whitelist)
	2014	+ free(options.whitelist);
	2015	+ if (options.blacklist)
	2016	+ free(options.blacklist);
	2017	+ if (cg_created)
	2018	+ cleanup_cgroup_environment();
1520	2019	close(cg_fd);
1521	2020	return err;
1522	2021	}