~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,35 +1,5 @@
1		-/*
2		- * Copyright (C) 2016-2018 Netronome Systems, Inc.
3		- *
4		- * This software is dual licensed under the GNU General License Version 2,
5		- * June 1991 as shown in the file COPYING in the top-level directory of this
6		- * source tree or the BSD 2-Clause License provided below. You have the
7		- * option to license this software under the complete terms of either license.
8		- *
9		- * The BSD 2-Clause License:
10		- *
11		- * Redistribution and use in source and binary forms, with or
12		- * without modification, are permitted provided that the following
13		- * conditions are met:
14		- *
15		- * 1. Redistributions of source code must retain the above
16		- * copyright notice, this list of conditions and the following
17		- * disclaimer.
18		- *
19		- * 2. Redistributions in binary form must reproduce the above
20		- * copyright notice, this list of conditions and the following
21		- * disclaimer in the documentation and/or other materials
22		- * provided with the distribution.
23		- *
24		- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25		- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26		- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27		- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28		- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29		- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30		- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31		- * SOFTWARE.
32		- */
	1	+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
	2	+/* Copyright (C) 2016-2018 Netronome Systems, Inc. */
33	3
34	4	#define pr_fmt(fmt) "NFP net bpf: " fmt
35	5
..	..	@@ -264,6 +234,38 @@
264	234	emit_br_bset(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, u8 defer)
265	235	{
266	236	emit_br_bit_relo(nfp_prog, src, bit, addr, defer, true, RELO_BR_REL);
	237	+}
	238	+
	239	+static void
	240	+__emit_br_alu(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
	241	+ u8 defer, bool dst_lmextn, bool src_lmextn)
	242	+{
	243	+ u64 insn;
	244	+
	245	+ insn = OP_BR_ALU_BASE \|
	246	+ FIELD_PREP(OP_BR_ALU_A_SRC, areg) \|
	247	+ FIELD_PREP(OP_BR_ALU_B_SRC, breg) \|
	248	+ FIELD_PREP(OP_BR_ALU_DEFBR, defer) \|
	249	+ FIELD_PREP(OP_BR_ALU_IMM_HI, imm_hi) \|
	250	+ FIELD_PREP(OP_BR_ALU_SRC_LMEXTN, src_lmextn) \|
	251	+ FIELD_PREP(OP_BR_ALU_DST_LMEXTN, dst_lmextn);
	252	+
	253	+ nfp_prog_push(nfp_prog, insn);
	254	+}
	255	+
	256	+static void emit_rtn(struct nfp_prog *nfp_prog, swreg base, u8 defer)
	257	+{
	258	+ struct nfp_insn_ur_regs reg;
	259	+ int err;
	260	+
	261	+ err = swreg_to_unrestricted(reg_none(), base, reg_imm(0), &reg);
	262	+ if (err) {
	263	+ nfp_prog->error = err;
	264	+ return;
	265	+ }
	266	+
	267	+ __emit_br_alu(nfp_prog, reg.areg, reg.breg, 0, defer, reg.dst_lmextn,
	268	+ reg.src_lmextn);
267	269	}
268	270
269	271	static void
..	..	@@ -621,6 +623,13 @@
621	623	}
622	624
623	625	static void
	626	+wrp_zext(struct nfp_prog nfp_prog, struct nfp_insn_meta meta, u8 dst)
	627	+{
	628	+ if (meta->flags & FLAG_INSN_DO_ZEXT)
	629	+ wrp_immed(nfp_prog, reg_both(dst + 1), 0);
	630	+}
	631	+
	632	+static void
624	633	wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm,
625	634	enum nfp_relo_type relo)
626	635	{
..	..	@@ -856,7 +865,8 @@
856	865	}
857	866
858	867	static int
859		-data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size)
	868	+data_ld(struct nfp_prog nfp_prog, struct nfp_insn_meta meta, swreg offset,
	869	+ u8 dst_gpr, int size)
860	870	{
861	871	unsigned int i;
862	872	u16 shift, sz;
..	..	@@ -879,14 +889,15 @@
879	889	wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
880	890
881	891	if (i < 2)
882		- wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0);
	892	+ wrp_zext(nfp_prog, meta, dst_gpr);
883	893
884	894	return 0;
885	895	}
886	896
887	897	static int
888		-data_ld_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr,
889		- swreg lreg, swreg rreg, int size, enum cmd_mode mode)
	898	+data_ld_host_order(struct nfp_prog nfp_prog, struct nfp_insn_meta meta,
	899	+ u8 dst_gpr, swreg lreg, swreg rreg, int size,
	900	+ enum cmd_mode mode)
890	901	{
891	902	unsigned int i;
892	903	u8 mask, sz;
..	..	@@ -909,33 +920,34 @@
909	920	wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
910	921
911	922	if (i < 2)
912		- wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0);
	923	+ wrp_zext(nfp_prog, meta, dst_gpr);
913	924
914	925	return 0;
915	926	}
916	927
917	928	static int
918		-data_ld_host_order_addr32(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
919		- u8 dst_gpr, u8 size)
	929	+data_ld_host_order_addr32(struct nfp_prog nfp_prog, struct nfp_insn_meta meta,
	930	+ u8 src_gpr, swreg offset, u8 dst_gpr, u8 size)
920	931	{
921		- return data_ld_host_order(nfp_prog, dst_gpr, reg_a(src_gpr), offset,
922		- size, CMD_MODE_32b);
	932	+ return data_ld_host_order(nfp_prog, meta, dst_gpr, reg_a(src_gpr),
	933	+ offset, size, CMD_MODE_32b);
923	934	}
924	935
925	936	static int
926		-data_ld_host_order_addr40(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
927		- u8 dst_gpr, u8 size)
	937	+data_ld_host_order_addr40(struct nfp_prog nfp_prog, struct nfp_insn_meta meta,
	938	+ u8 src_gpr, swreg offset, u8 dst_gpr, u8 size)
928	939	{
929	940	swreg rega, regb;
930	941
931	942	addr40_offset(nfp_prog, src_gpr, offset, &rega, &regb);
932	943
933		- return data_ld_host_order(nfp_prog, dst_gpr, rega, regb,
	944	+ return data_ld_host_order(nfp_prog, meta, dst_gpr, rega, regb,
934	945	size, CMD_MODE_40b_BA);
935	946	}
936	947
937	948	static int
938		-construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
	949	+construct_data_ind_ld(struct nfp_prog nfp_prog, struct nfp_insn_meta meta,
	950	+ u16 offset, u16 src, u8 size)
939	951	{
940	952	swreg tmp_reg;
941	953
..	..	@@ -951,10 +963,12 @@
951	963	emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);
952	964
953	965	/* Load data */
954		- return data_ld(nfp_prog, imm_b(nfp_prog), 0, size);
	966	+ return data_ld(nfp_prog, meta, imm_b(nfp_prog), 0, size);
955	967	}
956	968
957		-static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)
	969	+static int
	970	+construct_data_ld(struct nfp_prog nfp_prog, struct nfp_insn_meta meta,
	971	+ u16 offset, u8 size)
958	972	{
959	973	swreg tmp_reg;
960	974
..	..	@@ -965,7 +979,7 @@
965	979
966	980	/* Load data */
967	981	tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
968		- return data_ld(nfp_prog, tmp_reg, 0, size);
	982	+ return data_ld(nfp_prog, meta, tmp_reg, 0, size);
969	983	}
970	984
971	985	static int
..	..	@@ -1148,8 +1162,8 @@
1148	1162	unsigned int size, unsigned int ptr_off, u8 gpr, u8 ptr_gpr,
1149	1163	bool clr_gpr, lmem_step step)
1150	1164	{
1151		- s32 off = nfp_prog->stack_depth + meta->insn.off + ptr_off;
1152		- bool first = true, last;
	1165	+ s32 off = nfp_prog->stack_frame_depth + meta->insn.off + ptr_off;
	1166	+ bool first = true, narrow_ld, last;
1153	1167	bool needs_inc = false;
1154	1168	swreg stack_off_reg;
1155	1169	u8 prev_gpr = 255;
..	..	@@ -1157,7 +1171,8 @@
1157	1171	bool lm3 = true;
1158	1172	int ret;
1159	1173
1160		- if (meta->ptr_not_const) {
	1174	+ if (meta->ptr_not_const \|\|
	1175	+ meta->flags & FLAG_INSN_PTR_CALLER_STACK_FRAME) {
1161	1176	/* Use of the last encountered ptr_off is OK, they all have
1162	1177	* the same alignment. Depend on low bits of value being
1163	1178	* discarded when written to LMaddr register.
..	..	@@ -1194,14 +1209,23 @@
1194	1209
1195	1210	needs_inc = true;
1196	1211	}
	1212	+
	1213	+ narrow_ld = clr_gpr && size < 8;
	1214	+
1197	1215	if (lm3) {
	1216	+ unsigned int nop_cnt;
	1217	+
1198	1218	emit_csr_wr(nfp_prog, imm_b(nfp_prog), NFP_CSR_ACT_LM_ADDR3);
1199		- /* For size < 4 one slot will be filled by zeroing of upper. */
1200		- wrp_nops(nfp_prog, clr_gpr && size < 8 ? 2 : 3);
	1219	+ /* For size < 4 one slot will be filled by zeroing of upper,
	1220	+ * but be careful, that zeroing could be eliminated by zext
	1221	+ * optimization.
	1222	+ */
	1223	+ nop_cnt = narrow_ld && meta->flags & FLAG_INSN_DO_ZEXT ? 2 : 3;
	1224	+ wrp_nops(nfp_prog, nop_cnt);
1201	1225	}
1202	1226
1203		- if (clr_gpr && size < 8)
1204		- wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
	1227	+ if (narrow_ld)
	1228	+ wrp_zext(nfp_prog, meta, gpr);
1205	1229
1206	1230	while (size) {
1207	1231	u32 slice_end;
..	..	@@ -1274,7 +1298,7 @@
1274	1298	u64 imm = insn->imm; /* sign extend */
1275	1299
1276	1300	if (skip) {
1277		- meta->skip = true;
	1301	+ meta->flags \|= FLAG_INSN_SKIP_NOOP;
1278	1302	return 0;
1279	1303	}
1280	1304
..	..	@@ -1302,9 +1326,10 @@
1302	1326	enum alu_op alu_op)
1303	1327	{
1304	1328	const struct bpf_insn *insn = &meta->insn;
	1329	+ u8 dst = insn->dst_reg * 2;
1305	1330
1306		- wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm);
1307		- wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
	1331	+ wrp_alu_imm(nfp_prog, dst, alu_op, insn->imm);
	1332	+ wrp_zext(nfp_prog, meta, dst);
1308	1333
1309	1334	return 0;
1310	1335	}
..	..	@@ -1316,7 +1341,7 @@
1316	1341	u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
1317	1342
1318	1343	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
1319		- wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
	1344	+ wrp_zext(nfp_prog, meta, dst);
1320	1345
1321	1346	return 0;
1322	1347	}
..	..	@@ -1337,8 +1362,9 @@
1337	1362
1338	1363	wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op,
1339	1364	insn->src_reg * 2, br_mask, insn->off);
1340		- wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op,
1341		- insn->src_reg * 2 + 1, br_mask, insn->off);
	1365	+ if (is_mbpf_jmp64(meta))
	1366	+ wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op,
	1367	+ insn->src_reg * 2 + 1, br_mask, insn->off);
1342	1368
1343	1369	return 0;
1344	1370	}
..	..	@@ -1393,13 +1419,15 @@
1393	1419	else
1394	1420	emit_alu(nfp_prog, reg_none(), tmp_reg, alu_op, reg_a(reg));
1395	1421
1396		- tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
1397		- if (!code->swap)
1398		- emit_alu(nfp_prog, reg_none(),
1399		- reg_a(reg + 1), carry_op, tmp_reg);
1400		- else
1401		- emit_alu(nfp_prog, reg_none(),
1402		- tmp_reg, carry_op, reg_a(reg + 1));
	1422	+ if (is_mbpf_jmp64(meta)) {
	1423	+ tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
	1424	+ if (!code->swap)
	1425	+ emit_alu(nfp_prog, reg_none(),
	1426	+ reg_a(reg + 1), carry_op, tmp_reg);
	1427	+ else
	1428	+ emit_alu(nfp_prog, reg_none(),
	1429	+ tmp_reg, carry_op, reg_a(reg + 1));
	1430	+ }
1403	1431
1404	1432	emit_br(nfp_prog, code->br_mask, insn->off, 0);
1405	1433
..	..	@@ -1426,8 +1454,9 @@
1426	1454	}
1427	1455
1428	1456	emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg));
1429		- emit_alu(nfp_prog, reg_none(),
1430		- reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1));
	1457	+ if (is_mbpf_jmp64(meta))
	1458	+ emit_alu(nfp_prog, reg_none(),
	1459	+ reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1));
1431	1460	emit_br(nfp_prog, code->br_mask, insn->off, 0);
1432	1461
1433	1462	return 0;
..	..	@@ -1701,7 +1730,7 @@
1701	1730	s64 lm_off;
1702	1731
1703	1732	/* We only have to reload LM0 if the key is not at start of stack */
1704		- lm_off = nfp_prog->stack_depth;
	1733	+ lm_off = nfp_prog->stack_frame_depth;
1705	1734	lm_off += meta->arg2.reg.var_off.value + meta->arg2.reg.off;
1706	1735	load_lm_ptr = meta->arg2.var_off \|\| lm_off;
1707	1736
..	..	@@ -1814,10 +1843,10 @@
1814	1843	swreg stack_depth_reg;
1815	1844
1816	1845	stack_depth_reg = ur_load_imm_any(nfp_prog,
1817		- nfp_prog->stack_depth,
	1846	+ nfp_prog->stack_frame_depth,
1818	1847	stack_imm(nfp_prog));
1819		- emit_alu(nfp_prog, reg_both(dst),
1820		- stack_reg(nfp_prog), ALU_OP_ADD, stack_depth_reg);
	1848	+ emit_alu(nfp_prog, reg_both(dst), stack_reg(nfp_prog),
	1849	+ ALU_OP_ADD, stack_depth_reg);
1821	1850	wrp_immed(nfp_prog, reg_both(dst + 1), 0);
1822	1851	} else {
1823	1852	wrp_reg_mov(nfp_prog, dst, src);
..	..	@@ -1966,6 +1995,9 @@
1966	1995	*/
1967	1996	static int __shl_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
1968	1997	{
	1998	+ if (!shift_amt)
	1999	+ return 0;
	2000	+
1969	2001	if (shift_amt < 32) {
1970	2002	emit_shf(nfp_prog, reg_both(dst + 1), reg_a(dst + 1),
1971	2003	SHF_OP_NONE, reg_b(dst), SHF_SC_R_DSHF,
..	..	@@ -2078,6 +2110,9 @@
2078	2110	*/
2079	2111	static int __shr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
2080	2112	{
	2113	+ if (!shift_amt)
	2114	+ return 0;
	2115	+
2081	2116	if (shift_amt < 32) {
2082	2117	emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
2083	2118	reg_b(dst), SHF_SC_R_DSHF, shift_amt);
..	..	@@ -2179,6 +2214,9 @@
2179	2214	*/
2180	2215	static int __ashr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
2181	2216	{
	2217	+ if (!shift_amt)
	2218	+ return 0;
	2219	+
2182	2220	if (shift_amt < 32) {
2183	2221	emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
2184	2222	reg_b(dst), SHF_SC_R_DSHF, shift_amt);
..	..	@@ -2380,23 +2418,132 @@
2380	2418	u8 dst = meta->insn.dst_reg * 2;
2381	2419
2382	2420	emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst));
2383		- wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
	2421	+ wrp_zext(nfp_prog, meta, dst);
2384	2422
	2423	+ return 0;
	2424	+}
	2425	+
	2426	+static int
	2427	+__ashr_imm(struct nfp_prog nfp_prog, struct nfp_insn_meta meta, u8 dst,
	2428	+ u8 shift_amt)
	2429	+{
	2430	+ if (shift_amt) {
	2431	+ /* Set signedness bit (MSB of result). */
	2432	+ emit_alu(nfp_prog, reg_none(), reg_a(dst), ALU_OP_OR,
	2433	+ reg_imm(0));
	2434	+ emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
	2435	+ reg_b(dst), SHF_SC_R_SHF, shift_amt);
	2436	+ }
	2437	+ wrp_zext(nfp_prog, meta, dst);
	2438	+
	2439	+ return 0;
	2440	+}
	2441	+
	2442	+static int ashr_reg(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	2443	+{
	2444	+ const struct bpf_insn *insn = &meta->insn;
	2445	+ u64 umin, umax;
	2446	+ u8 dst, src;
	2447	+
	2448	+ dst = insn->dst_reg * 2;
	2449	+ umin = meta->umin_src;
	2450	+ umax = meta->umax_src;
	2451	+ if (umin == umax)
	2452	+ return __ashr_imm(nfp_prog, meta, dst, umin);
	2453	+
	2454	+ src = insn->src_reg * 2;
	2455	+ /* NOTE: the first insn will set both indirect shift amount (source A)
	2456	+ * and signedness bit (MSB of result).
	2457	+ */
	2458	+ emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst));
	2459	+ emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
	2460	+ reg_b(dst), SHF_SC_R_SHF);
	2461	+ wrp_zext(nfp_prog, meta, dst);
	2462	+
	2463	+ return 0;
	2464	+}
	2465	+
	2466	+static int ashr_imm(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	2467	+{
	2468	+ const struct bpf_insn *insn = &meta->insn;
	2469	+ u8 dst = insn->dst_reg * 2;
	2470	+
	2471	+ return __ashr_imm(nfp_prog, meta, dst, insn->imm);
	2472	+}
	2473	+
	2474	+static int
	2475	+__shr_imm(struct nfp_prog nfp_prog, struct nfp_insn_meta meta, u8 dst,
	2476	+ u8 shift_amt)
	2477	+{
	2478	+ if (shift_amt)
	2479	+ emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
	2480	+ reg_b(dst), SHF_SC_R_SHF, shift_amt);
	2481	+ wrp_zext(nfp_prog, meta, dst);
	2482	+ return 0;
	2483	+}
	2484	+
	2485	+static int shr_imm(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	2486	+{
	2487	+ const struct bpf_insn *insn = &meta->insn;
	2488	+ u8 dst = insn->dst_reg * 2;
	2489	+
	2490	+ return __shr_imm(nfp_prog, meta, dst, insn->imm);
	2491	+}
	2492	+
	2493	+static int shr_reg(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	2494	+{
	2495	+ const struct bpf_insn *insn = &meta->insn;
	2496	+ u64 umin, umax;
	2497	+ u8 dst, src;
	2498	+
	2499	+ dst = insn->dst_reg * 2;
	2500	+ umin = meta->umin_src;
	2501	+ umax = meta->umax_src;
	2502	+ if (umin == umax)
	2503	+ return __shr_imm(nfp_prog, meta, dst, umin);
	2504	+
	2505	+ src = insn->src_reg * 2;
	2506	+ emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
	2507	+ emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
	2508	+ reg_b(dst), SHF_SC_R_SHF);
	2509	+ wrp_zext(nfp_prog, meta, dst);
	2510	+ return 0;
	2511	+}
	2512	+
	2513	+static int
	2514	+__shl_imm(struct nfp_prog nfp_prog, struct nfp_insn_meta meta, u8 dst,
	2515	+ u8 shift_amt)
	2516	+{
	2517	+ if (shift_amt)
	2518	+ emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
	2519	+ reg_b(dst), SHF_SC_L_SHF, shift_amt);
	2520	+ wrp_zext(nfp_prog, meta, dst);
2385	2521	return 0;
2386	2522	}
2387	2523
2388	2524	static int shl_imm(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
2389	2525	{
2390	2526	const struct bpf_insn *insn = &meta->insn;
	2527	+ u8 dst = insn->dst_reg * 2;
2391	2528
2392		- if (!insn->imm)
2393		- return 1; /* TODO: zero shift means indirect */
	2529	+ return __shl_imm(nfp_prog, meta, dst, insn->imm);
	2530	+}
2394	2531
2395		- emit_shf(nfp_prog, reg_both(insn->dst_reg * 2),
2396		- reg_none(), SHF_OP_NONE, reg_b(insn->dst_reg * 2),
2397		- SHF_SC_L_SHF, insn->imm);
2398		- wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
	2532	+static int shl_reg(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	2533	+{
	2534	+ const struct bpf_insn *insn = &meta->insn;
	2535	+ u64 umin, umax;
	2536	+ u8 dst, src;
2399	2537
	2538	+ dst = insn->dst_reg * 2;
	2539	+ umin = meta->umin_src;
	2540	+ umax = meta->umax_src;
	2541	+ if (umin == umax)
	2542	+ return __shl_imm(nfp_prog, meta, dst, umin);
	2543	+
	2544	+ src = insn->src_reg * 2;
	2545	+ shl_reg64_lt32_low(nfp_prog, dst, src);
	2546	+ wrp_zext(nfp_prog, meta, dst);
2400	2547	return 0;
2401	2548	}
2402	2549
..	..	@@ -2458,34 +2605,34 @@
2458	2605
2459	2606	static int data_ld1(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
2460	2607	{
2461		- return construct_data_ld(nfp_prog, meta->insn.imm, 1);
	2608	+ return construct_data_ld(nfp_prog, meta, meta->insn.imm, 1);
2462	2609	}
2463	2610
2464	2611	static int data_ld2(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
2465	2612	{
2466		- return construct_data_ld(nfp_prog, meta->insn.imm, 2);
	2613	+ return construct_data_ld(nfp_prog, meta, meta->insn.imm, 2);
2467	2614	}
2468	2615
2469	2616	static int data_ld4(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
2470	2617	{
2471		- return construct_data_ld(nfp_prog, meta->insn.imm, 4);
	2618	+ return construct_data_ld(nfp_prog, meta, meta->insn.imm, 4);
2472	2619	}
2473	2620
2474	2621	static int data_ind_ld1(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
2475	2622	{
2476		- return construct_data_ind_ld(nfp_prog, meta->insn.imm,
	2623	+ return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
2477	2624	meta->insn.src_reg * 2, 1);
2478	2625	}
2479	2626
2480	2627	static int data_ind_ld2(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
2481	2628	{
2482		- return construct_data_ind_ld(nfp_prog, meta->insn.imm,
	2629	+ return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
2483	2630	meta->insn.src_reg * 2, 2);
2484	2631	}
2485	2632
2486	2633	static int data_ind_ld4(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
2487	2634	{
2488		- return construct_data_ind_ld(nfp_prog, meta->insn.imm,
	2635	+ return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
2489	2636	meta->insn.src_reg * 2, 4);
2490	2637	}
2491	2638
..	..	@@ -2505,17 +2652,17 @@
2505	2652
2506	2653	switch (meta->insn.off) {
2507	2654	case offsetof(struct __sk_buff, len):
2508		- if (size != FIELD_SIZEOF(struct __sk_buff, len))
	2655	+ if (size != sizeof_field(struct __sk_buff, len))
2509	2656	return -EOPNOTSUPP;
2510	2657	wrp_mov(nfp_prog, dst, plen_reg(nfp_prog));
2511	2658	break;
2512	2659	case offsetof(struct __sk_buff, data):
2513		- if (size != FIELD_SIZEOF(struct __sk_buff, data))
	2660	+ if (size != sizeof_field(struct __sk_buff, data))
2514	2661	return -EOPNOTSUPP;
2515	2662	wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
2516	2663	break;
2517	2664	case offsetof(struct __sk_buff, data_end):
2518		- if (size != FIELD_SIZEOF(struct __sk_buff, data_end))
	2665	+ if (size != sizeof_field(struct __sk_buff, data_end))
2519	2666	return -EOPNOTSUPP;
2520	2667	emit_alu(nfp_prog, dst,
2521	2668	plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
..	..	@@ -2536,12 +2683,12 @@
2536	2683
2537	2684	switch (meta->insn.off) {
2538	2685	case offsetof(struct xdp_md, data):
2539		- if (size != FIELD_SIZEOF(struct xdp_md, data))
	2686	+ if (size != sizeof_field(struct xdp_md, data))
2540	2687	return -EOPNOTSUPP;
2541	2688	wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
2542	2689	break;
2543	2690	case offsetof(struct xdp_md, data_end):
2544		- if (size != FIELD_SIZEOF(struct xdp_md, data_end))
	2691	+ if (size != sizeof_field(struct xdp_md, data_end))
2545	2692	return -EOPNOTSUPP;
2546	2693	emit_alu(nfp_prog, dst,
2547	2694	plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
..	..	@@ -2563,7 +2710,7 @@
2563	2710
2564	2711	tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
2565	2712
2566		- return data_ld_host_order_addr32(nfp_prog, meta->insn.src_reg * 2,
	2713	+ return data_ld_host_order_addr32(nfp_prog, meta, meta->insn.src_reg * 2,
2567	2714	tmp_reg, meta->insn.dst_reg * 2, size);
2568	2715	}
2569	2716
..	..	@@ -2575,7 +2722,7 @@
2575	2722
2576	2723	tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
2577	2724
2578		- return data_ld_host_order_addr40(nfp_prog, meta->insn.src_reg * 2,
	2725	+ return data_ld_host_order_addr40(nfp_prog, meta, meta->insn.src_reg * 2,
2579	2726	tmp_reg, meta->insn.dst_reg * 2, size);
2580	2727	}
2581	2728
..	..	@@ -2636,7 +2783,7 @@
2636	2783	wrp_reg_subpart(nfp_prog, dst_lo, src_lo, len_lo, off);
2637	2784
2638	2785	if (!len_mid) {
2639		- wrp_immed(nfp_prog, dst_hi, 0);
	2786	+ wrp_zext(nfp_prog, meta, dst_gpr);
2640	2787	return 0;
2641	2788	}
2642	2789
..	..	@@ -2644,7 +2791,7 @@
2644	2791
2645	2792	if (size <= REG_WIDTH) {
2646	2793	wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, len_mid, len_lo);
2647		- wrp_immed(nfp_prog, dst_hi, 0);
	2794	+ wrp_zext(nfp_prog, meta, dst_gpr);
2648	2795	} else {
2649	2796	swreg src_hi = reg_xfer(idx + 2);
2650	2797
..	..	@@ -2675,10 +2822,10 @@
2675	2822
2676	2823	if (size < REG_WIDTH) {
2677	2824	wrp_reg_subpart(nfp_prog, dst_lo, src_lo, size, 0);
2678		- wrp_immed(nfp_prog, dst_hi, 0);
	2825	+ wrp_zext(nfp_prog, meta, dst_gpr);
2679	2826	} else if (size == REG_WIDTH) {
2680	2827	wrp_mov(nfp_prog, dst_lo, src_lo);
2681		- wrp_immed(nfp_prog, dst_hi, 0);
	2828	+ wrp_zext(nfp_prog, meta, dst_gpr);
2682	2829	} else {
2683	2830	swreg src_hi = reg_xfer(idx + 1);
2684	2831
..	..	@@ -3008,30 +3155,37 @@
3008	3155	return 0;
3009	3156	}
3010	3157
	3158	+static int jeq32_imm(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	3159	+{
	3160	+ const struct bpf_insn *insn = &meta->insn;
	3161	+ swreg tmp_reg;
	3162	+
	3163	+ tmp_reg = ur_load_imm_any(nfp_prog, insn->imm, imm_b(nfp_prog));
	3164	+ emit_alu(nfp_prog, reg_none(),
	3165	+ reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
	3166	+ emit_br(nfp_prog, BR_BEQ, insn->off, 0);
	3167	+
	3168	+ return 0;
	3169	+}
	3170	+
3011	3171	static int jset_imm(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
3012	3172	{
3013	3173	const struct bpf_insn *insn = &meta->insn;
3014	3174	u64 imm = insn->imm; /* sign extend */
	3175	+ u8 dst_gpr = insn->dst_reg * 2;
3015	3176	swreg tmp_reg;
3016	3177
3017		- if (!imm) {
3018		- meta->skip = true;
3019		- return 0;
3020		- }
3021		-
3022		- if (imm & ~0U) {
3023		- tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
	3178	+ tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
	3179	+ emit_alu(nfp_prog, imm_b(nfp_prog),
	3180	+ reg_a(dst_gpr), ALU_OP_AND, tmp_reg);
	3181	+ /* Upper word of the mask can only be 0 or ~0 from sign extension,
	3182	+ * so either ignore it or OR the whole thing in.
	3183	+ */
	3184	+ if (is_mbpf_jmp64(meta) && imm >> 32) {
3024	3185	emit_alu(nfp_prog, reg_none(),
3025		- reg_a(insn->dst_reg * 2), ALU_OP_AND, tmp_reg);
3026		- emit_br(nfp_prog, BR_BNE, insn->off, 0);
	3186	+ reg_a(dst_gpr + 1), ALU_OP_OR, imm_b(nfp_prog));
3027	3187	}
3028		-
3029		- if (imm >> 32) {
3030		- tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
3031		- emit_alu(nfp_prog, reg_none(),
3032		- reg_a(insn->dst_reg * 2 + 1), ALU_OP_AND, tmp_reg);
3033		- emit_br(nfp_prog, BR_BNE, insn->off, 0);
3034		- }
	3188	+ emit_br(nfp_prog, BR_BNE, insn->off, 0);
3035	3189
3036	3190	return 0;
3037	3191	}
..	..	@@ -3040,11 +3194,16 @@
3040	3194	{
3041	3195	const struct bpf_insn *insn = &meta->insn;
3042	3196	u64 imm = insn->imm; /* sign extend */
	3197	+ bool is_jmp32 = is_mbpf_jmp32(meta);
3043	3198	swreg tmp_reg;
3044	3199
3045	3200	if (!imm) {
3046		- emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2),
3047		- ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1));
	3201	+ if (is_jmp32)
	3202	+ emit_alu(nfp_prog, reg_none(), reg_none(), ALU_OP_NONE,
	3203	+ reg_b(insn->dst_reg * 2));
	3204	+ else
	3205	+ emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2),
	3206	+ ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1));
3048	3207	emit_br(nfp_prog, BR_BNE, insn->off, 0);
3049	3208	return 0;
3050	3209	}
..	..	@@ -3053,6 +3212,9 @@
3053	3212	emit_alu(nfp_prog, reg_none(),
3054	3213	reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
3055	3214	emit_br(nfp_prog, BR_BNE, insn->off, 0);
	3215	+
	3216	+ if (is_jmp32)
	3217	+ return 0;
3056	3218
3057	3219	tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
3058	3220	emit_alu(nfp_prog, reg_none(),
..	..	@@ -3068,10 +3230,13 @@
3068	3230
3069	3231	emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2),
3070	3232	ALU_OP_XOR, reg_b(insn->src_reg * 2));
3071		- emit_alu(nfp_prog, imm_b(nfp_prog), reg_a(insn->dst_reg * 2 + 1),
3072		- ALU_OP_XOR, reg_b(insn->src_reg * 2 + 1));
3073		- emit_alu(nfp_prog, reg_none(),
3074		- imm_a(nfp_prog), ALU_OP_OR, imm_b(nfp_prog));
	3233	+ if (is_mbpf_jmp64(meta)) {
	3234	+ emit_alu(nfp_prog, imm_b(nfp_prog),
	3235	+ reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR,
	3236	+ reg_b(insn->src_reg * 2 + 1));
	3237	+ emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR,
	3238	+ imm_b(nfp_prog));
	3239	+ }
3075	3240	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
3076	3241
3077	3242	return 0;
..	..	@@ -3087,7 +3252,93 @@
3087	3252	return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE);
3088	3253	}
3089	3254
3090		-static int call(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	3255	+static int
	3256	+bpf_to_bpf_call(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	3257	+{
	3258	+ u32 ret_tgt, stack_depth, offset_br;
	3259	+ swreg tmp_reg;
	3260	+
	3261	+ stack_depth = round_up(nfp_prog->stack_frame_depth, STACK_FRAME_ALIGN);
	3262	+ /* Space for saving the return address is accounted for by the callee,
	3263	+ * so stack_depth can be zero for the main function.
	3264	+ */
	3265	+ if (stack_depth) {
	3266	+ tmp_reg = ur_load_imm_any(nfp_prog, stack_depth,
	3267	+ stack_imm(nfp_prog));
	3268	+ emit_alu(nfp_prog, stack_reg(nfp_prog),
	3269	+ stack_reg(nfp_prog), ALU_OP_ADD, tmp_reg);
	3270	+ emit_csr_wr(nfp_prog, stack_reg(nfp_prog),
	3271	+ NFP_CSR_ACT_LM_ADDR0);
	3272	+ }
	3273	+
	3274	+ /* Two cases for jumping to the callee:
	3275	+ *
	3276	+ * - If callee uses and needs to save R6~R9 then:
	3277	+ * 1. Put the start offset of the callee into imm_b(). This will
	3278	+ * require a fixup step, as we do not necessarily know this
	3279	+ * address yet.
	3280	+ * 2. Put the return address from the callee to the caller into
	3281	+ * register ret_reg().
	3282	+ * 3. (After defer slots are consumed) Jump to the subroutine that
	3283	+ * pushes the registers to the stack.
	3284	+ * The subroutine acts as a trampoline, and returns to the address in
	3285	+ * imm_b(), i.e. jumps to the callee.
	3286	+ *
	3287	+ * - If callee does not need to save R6~R9 then just load return
	3288	+ * address to the caller in ret_reg(), and jump to the callee
	3289	+ * directly.
	3290	+ *
	3291	+ * Using ret_reg() to pass the return address to the callee is set here
	3292	+ * as a convention. The callee can then push this address onto its
	3293	+ * stack frame in its prologue. The advantages of passing the return
	3294	+ * address through ret_reg(), instead of pushing it to the stack right
	3295	+ * here, are the following:
	3296	+ * - It looks cleaner.
	3297	+ * - If the called function is called multiple time, we get a lower
	3298	+ * program size.
	3299	+ * - We save two no-op instructions that should be added just before
	3300	+ * the emit_br() when stack depth is not null otherwise.
	3301	+ * - If we ever find a register to hold the return address during whole
	3302	+ * execution of the callee, we will not have to push the return
	3303	+ * address to the stack for leaf functions.
	3304	+ */
	3305	+ if (!meta->jmp_dst) {
	3306	+ pr_err("BUG: BPF-to-BPF call has no destination recorded\n");
	3307	+ return -ELOOP;
	3308	+ }
	3309	+ if (nfp_prog->subprog[meta->jmp_dst->subprog_idx].needs_reg_push) {
	3310	+ ret_tgt = nfp_prog_current_offset(nfp_prog) + 3;
	3311	+ emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2,
	3312	+ RELO_BR_GO_CALL_PUSH_REGS);
	3313	+ offset_br = nfp_prog_current_offset(nfp_prog);
	3314	+ wrp_immed_relo(nfp_prog, imm_b(nfp_prog), 0, RELO_IMMED_REL);
	3315	+ } else {
	3316	+ ret_tgt = nfp_prog_current_offset(nfp_prog) + 2;
	3317	+ emit_br(nfp_prog, BR_UNC, meta->insn.imm, 1);
	3318	+ offset_br = nfp_prog_current_offset(nfp_prog);
	3319	+ }
	3320	+ wrp_immed_relo(nfp_prog, ret_reg(nfp_prog), ret_tgt, RELO_IMMED_REL);
	3321	+
	3322	+ if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
	3323	+ return -EINVAL;
	3324	+
	3325	+ if (stack_depth) {
	3326	+ tmp_reg = ur_load_imm_any(nfp_prog, stack_depth,
	3327	+ stack_imm(nfp_prog));
	3328	+ emit_alu(nfp_prog, stack_reg(nfp_prog),
	3329	+ stack_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
	3330	+ emit_csr_wr(nfp_prog, stack_reg(nfp_prog),
	3331	+ NFP_CSR_ACT_LM_ADDR0);
	3332	+ wrp_nops(nfp_prog, 3);
	3333	+ }
	3334	+
	3335	+ meta->num_insns_after_br = nfp_prog_current_offset(nfp_prog);
	3336	+ meta->num_insns_after_br -= offset_br;
	3337	+
	3338	+ return 0;
	3339	+}
	3340	+
	3341	+static int helper_call(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
3091	3342	{
3092	3343	switch (meta->insn.imm) {
3093	3344	case BPF_FUNC_xdp_adjust_head:
..	..	@@ -3108,11 +3359,57 @@
3108	3359	}
3109	3360	}
3110	3361
	3362	+static int call(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	3363	+{
	3364	+ if (is_mbpf_pseudo_call(meta))
	3365	+ return bpf_to_bpf_call(nfp_prog, meta);
	3366	+ else
	3367	+ return helper_call(nfp_prog, meta);
	3368	+}
	3369	+
	3370	+static bool nfp_is_main_function(struct nfp_insn_meta *meta)
	3371	+{
	3372	+ return meta->subprog_idx == 0;
	3373	+}
	3374	+
3111	3375	static int goto_out(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
3112	3376	{
3113	3377	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 0, RELO_BR_GO_OUT);
3114	3378
3115	3379	return 0;
	3380	+}
	3381	+
	3382	+static int
	3383	+nfp_subprog_epilogue(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	3384	+{
	3385	+ if (nfp_prog->subprog[meta->subprog_idx].needs_reg_push) {
	3386	+ /* Pop R6~R9 to the stack via related subroutine.
	3387	+ * We loaded the return address to the caller into ret_reg().
	3388	+ * This means that the subroutine does not come back here, we
	3389	+ * make it jump back to the subprogram caller directly!
	3390	+ */
	3391	+ emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 1,
	3392	+ RELO_BR_GO_CALL_POP_REGS);
	3393	+ /* Pop return address from the stack. */
	3394	+ wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0));
	3395	+ } else {
	3396	+ /* Pop return address from the stack. */
	3397	+ wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0));
	3398	+ /* Jump back to caller if no callee-saved registers were used
	3399	+ * by the subprogram.
	3400	+ */
	3401	+ emit_rtn(nfp_prog, ret_reg(nfp_prog), 0);
	3402	+ }
	3403	+
	3404	+ return 0;
	3405	+}
	3406	+
	3407	+static int jmp_exit(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	3408	+{
	3409	+ if (nfp_is_main_function(meta))
	3410	+ return goto_out(nfp_prog, meta);
	3411	+ else
	3412	+ return nfp_subprog_epilogue(nfp_prog, meta);
3116	3413	}
3117	3414
3118	3415	static const instr_cb_t instr_cb[256] = {
..	..	@@ -3156,7 +3453,12 @@
3156	3453	[BPF_ALU \| BPF_DIV \| BPF_X] = div_reg,
3157	3454	[BPF_ALU \| BPF_DIV \| BPF_K] = div_imm,
3158	3455	[BPF_ALU \| BPF_NEG] = neg_reg,
	3456	+ [BPF_ALU \| BPF_LSH \| BPF_X] = shl_reg,
3159	3457	[BPF_ALU \| BPF_LSH \| BPF_K] = shl_imm,
	3458	+ [BPF_ALU \| BPF_RSH \| BPF_X] = shr_reg,
	3459	+ [BPF_ALU \| BPF_RSH \| BPF_K] = shr_imm,
	3460	+ [BPF_ALU \| BPF_ARSH \| BPF_X] = ashr_reg,
	3461	+ [BPF_ALU \| BPF_ARSH \| BPF_K] = ashr_imm,
3160	3462	[BPF_ALU \| BPF_END \| BPF_X] = end_reg32,
3161	3463	[BPF_LD \| BPF_IMM \| BPF_DW] = imm_ld8,
3162	3464	[BPF_LD \| BPF_ABS \| BPF_B] = data_ld1,
..	..	@@ -3202,22 +3504,62 @@
3202	3504	[BPF_JMP \| BPF_JSLE \| BPF_X] = cmp_reg,
3203	3505	[BPF_JMP \| BPF_JSET \| BPF_X] = jset_reg,
3204	3506	[BPF_JMP \| BPF_JNE \| BPF_X] = jne_reg,
	3507	+ [BPF_JMP32 \| BPF_JEQ \| BPF_K] = jeq32_imm,
	3508	+ [BPF_JMP32 \| BPF_JGT \| BPF_K] = cmp_imm,
	3509	+ [BPF_JMP32 \| BPF_JGE \| BPF_K] = cmp_imm,
	3510	+ [BPF_JMP32 \| BPF_JLT \| BPF_K] = cmp_imm,
	3511	+ [BPF_JMP32 \| BPF_JLE \| BPF_K] = cmp_imm,
	3512	+ [BPF_JMP32 \| BPF_JSGT \| BPF_K] =cmp_imm,
	3513	+ [BPF_JMP32 \| BPF_JSGE \| BPF_K] =cmp_imm,
	3514	+ [BPF_JMP32 \| BPF_JSLT \| BPF_K] =cmp_imm,
	3515	+ [BPF_JMP32 \| BPF_JSLE \| BPF_K] =cmp_imm,
	3516	+ [BPF_JMP32 \| BPF_JSET \| BPF_K] =jset_imm,
	3517	+ [BPF_JMP32 \| BPF_JNE \| BPF_K] = jne_imm,
	3518	+ [BPF_JMP32 \| BPF_JEQ \| BPF_X] = jeq_reg,
	3519	+ [BPF_JMP32 \| BPF_JGT \| BPF_X] = cmp_reg,
	3520	+ [BPF_JMP32 \| BPF_JGE \| BPF_X] = cmp_reg,
	3521	+ [BPF_JMP32 \| BPF_JLT \| BPF_X] = cmp_reg,
	3522	+ [BPF_JMP32 \| BPF_JLE \| BPF_X] = cmp_reg,
	3523	+ [BPF_JMP32 \| BPF_JSGT \| BPF_X] =cmp_reg,
	3524	+ [BPF_JMP32 \| BPF_JSGE \| BPF_X] =cmp_reg,
	3525	+ [BPF_JMP32 \| BPF_JSLT \| BPF_X] =cmp_reg,
	3526	+ [BPF_JMP32 \| BPF_JSLE \| BPF_X] =cmp_reg,
	3527	+ [BPF_JMP32 \| BPF_JSET \| BPF_X] =jset_reg,
	3528	+ [BPF_JMP32 \| BPF_JNE \| BPF_X] = jne_reg,
3205	3529	[BPF_JMP \| BPF_CALL] = call,
3206		- [BPF_JMP \| BPF_EXIT] = goto_out,
	3530	+ [BPF_JMP \| BPF_EXIT] = jmp_exit,
3207	3531	};
3208	3532
3209	3533	/* --- Assembler logic --- */
	3534	+static int
	3535	+nfp_fixup_immed_relo(struct nfp_prog nfp_prog, struct nfp_insn_meta meta,
	3536	+ struct nfp_insn_meta *jmp_dst, u32 br_idx)
	3537	+{
	3538	+ if (immed_get_value(nfp_prog->prog[br_idx + 1])) {
	3539	+ pr_err("BUG: failed to fix up callee register saving\n");
	3540	+ return -EINVAL;
	3541	+ }
	3542	+
	3543	+ immed_set_value(&nfp_prog->prog[br_idx + 1], jmp_dst->off);
	3544	+
	3545	+ return 0;
	3546	+}
	3547	+
3210	3548	static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
3211	3549	{
3212	3550	struct nfp_insn_meta meta, jmp_dst;
3213	3551	u32 idx, br_idx;
	3552	+ int err;
3214	3553
3215	3554	list_for_each_entry(meta, &nfp_prog->insns, l) {
3216		- if (meta->skip)
	3555	+ if (meta->flags & FLAG_INSN_SKIP_MASK)
3217	3556	continue;
3218		- if (meta->insn.code == (BPF_JMP \| BPF_CALL))
	3557	+ if (!is_mbpf_jmp(meta))
3219	3558	continue;
3220		- if (BPF_CLASS(meta->insn.code) != BPF_JMP)
	3559	+ if (meta->insn.code == (BPF_JMP \| BPF_EXIT) &&
	3560	+ !nfp_is_main_function(meta))
	3561	+ continue;
	3562	+ if (is_mbpf_helper_call(meta))
3221	3563	continue;
3222	3564
3223	3565	if (list_is_last(&meta->l, &nfp_prog->insns))
..	..	@@ -3225,14 +3567,26 @@
3225	3567	else
3226	3568	br_idx = list_next_entry(meta, l)->off - 1;
3227	3569
	3570	+ /* For BPF-to-BPF function call, a stack adjustment sequence is
	3571	+ * generated after the return instruction. Therefore, we must
	3572	+ * withdraw the length of this sequence to have br_idx pointing
	3573	+ * to where the "branch" NFP instruction is expected to be.
	3574	+ */
	3575	+ if (is_mbpf_pseudo_call(meta))
	3576	+ br_idx -= meta->num_insns_after_br;
	3577	+
3228	3578	if (!nfp_is_br(nfp_prog->prog[br_idx])) {
3229	3579	pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n",
3230	3580	br_idx, meta->insn.code, nfp_prog->prog[br_idx]);
3231	3581	return -ELOOP;
3232	3582	}
	3583	+
	3584	+ if (meta->insn.code == (BPF_JMP \| BPF_EXIT))
	3585	+ continue;
	3586	+
3233	3587	/* Leave special branches for later */
3234	3588	if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
3235		- RELO_BR_REL)
	3589	+ RELO_BR_REL && !is_mbpf_pseudo_call(meta))
3236	3590	continue;
3237	3591
3238	3592	if (!meta->jmp_dst) {
..	..	@@ -3242,10 +3596,22 @@
3242	3596
3243	3597	jmp_dst = meta->jmp_dst;
3244	3598
3245		- if (jmp_dst->skip) {
	3599	+ if (jmp_dst->flags & FLAG_INSN_SKIP_PREC_DEPENDENT) {
3246	3600	pr_err("Branch landing on removed instruction!!\n");
3247	3601	return -ELOOP;
3248	3602	}
	3603	+
	3604	+ if (is_mbpf_pseudo_call(meta) &&
	3605	+ nfp_prog->subprog[jmp_dst->subprog_idx].needs_reg_push) {
	3606	+ err = nfp_fixup_immed_relo(nfp_prog, meta,
	3607	+ jmp_dst, br_idx);
	3608	+ if (err)
	3609	+ return err;
	3610	+ }
	3611	+
	3612	+ if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
	3613	+ RELO_BR_REL)
	3614	+ continue;
3249	3615
3250	3616	for (idx = meta->off; idx <= br_idx; idx++) {
3251	3617	if (!nfp_is_br(nfp_prog->prog[idx]))
..	..	@@ -3262,6 +3628,27 @@
3262	3628	wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0));
3263	3629	emit_alu(nfp_prog, plen_reg(nfp_prog),
3264	3630	plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog));
	3631	+}
	3632	+
	3633	+static void
	3634	+nfp_subprog_prologue(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	3635	+{
	3636	+ /* Save return address into the stack. */
	3637	+ wrp_mov(nfp_prog, reg_lm(0, 0), ret_reg(nfp_prog));
	3638	+}
	3639	+
	3640	+static void
	3641	+nfp_start_subprog(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
	3642	+{
	3643	+ unsigned int depth = nfp_prog->subprog[meta->subprog_idx].stack_depth;
	3644	+
	3645	+ nfp_prog->stack_frame_depth = round_up(depth, 4);
	3646	+ nfp_subprog_prologue(nfp_prog, meta);
	3647	+}
	3648	+
	3649	+bool nfp_is_subprog_start(struct nfp_insn_meta *meta)
	3650	+{
	3651	+ return meta->flags & FLAG_INSN_IS_SUBPROG_START;
3265	3652	}
3266	3653
3267	3654	static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
..	..	@@ -3354,6 +3741,67 @@
3354	3741	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
3355	3742	}
3356	3743
	3744	+static bool nfp_prog_needs_callee_reg_save(struct nfp_prog *nfp_prog)
	3745	+{
	3746	+ unsigned int idx;
	3747	+
	3748	+ for (idx = 1; idx < nfp_prog->subprog_cnt; idx++)
	3749	+ if (nfp_prog->subprog[idx].needs_reg_push)
	3750	+ return true;
	3751	+
	3752	+ return false;
	3753	+}
	3754	+
	3755	+static void nfp_push_callee_registers(struct nfp_prog *nfp_prog)
	3756	+{
	3757	+ u8 reg;
	3758	+
	3759	+ /* Subroutine: Save all callee saved registers (R6 ~ R9).
	3760	+ * imm_b() holds the return address.
	3761	+ */
	3762	+ nfp_prog->tgt_call_push_regs = nfp_prog_current_offset(nfp_prog);
	3763	+ for (reg = BPF_REG_6; reg <= BPF_REG_9; reg++) {
	3764	+ u8 adj = (reg - BPF_REG_0) * 2;
	3765	+ u8 idx = (reg - BPF_REG_6) * 2;
	3766	+
	3767	+ /* The first slot in the stack frame is used to push the return
	3768	+ * address in bpf_to_bpf_call(), start just after.
	3769	+ */
	3770	+ wrp_mov(nfp_prog, reg_lm(0, 1 + idx), reg_b(adj));
	3771	+
	3772	+ if (reg == BPF_REG_8)
	3773	+ /* Prepare to jump back, last 3 insns use defer slots */
	3774	+ emit_rtn(nfp_prog, imm_b(nfp_prog), 3);
	3775	+
	3776	+ wrp_mov(nfp_prog, reg_lm(0, 1 + idx + 1), reg_b(adj + 1));
	3777	+ }
	3778	+}
	3779	+
	3780	+static void nfp_pop_callee_registers(struct nfp_prog *nfp_prog)
	3781	+{
	3782	+ u8 reg;
	3783	+
	3784	+ /* Subroutine: Restore all callee saved registers (R6 ~ R9).
	3785	+ * ret_reg() holds the return address.
	3786	+ */
	3787	+ nfp_prog->tgt_call_pop_regs = nfp_prog_current_offset(nfp_prog);
	3788	+ for (reg = BPF_REG_6; reg <= BPF_REG_9; reg++) {
	3789	+ u8 adj = (reg - BPF_REG_0) * 2;
	3790	+ u8 idx = (reg - BPF_REG_6) * 2;
	3791	+
	3792	+ /* The first slot in the stack frame holds the return address,
	3793	+ * start popping just after that.
	3794	+ */
	3795	+ wrp_mov(nfp_prog, reg_both(adj), reg_lm(0, 1 + idx));
	3796	+
	3797	+ if (reg == BPF_REG_8)
	3798	+ /* Prepare to jump back, last 3 insns use defer slots */
	3799	+ emit_rtn(nfp_prog, ret_reg(nfp_prog), 3);
	3800	+
	3801	+ wrp_mov(nfp_prog, reg_both(adj + 1), reg_lm(0, 1 + idx + 1));
	3802	+ }
	3803	+}
	3804	+
3357	3805	static void nfp_outro(struct nfp_prog *nfp_prog)
3358	3806	{
3359	3807	switch (nfp_prog->type) {
..	..	@@ -3366,12 +3814,22 @@
3366	3814	default:
3367	3815	WARN_ON(1);
3368	3816	}
	3817	+
	3818	+ if (!nfp_prog_needs_callee_reg_save(nfp_prog))
	3819	+ return;
	3820	+
	3821	+ nfp_push_callee_registers(nfp_prog);
	3822	+ nfp_pop_callee_registers(nfp_prog);
3369	3823	}
3370	3824
3371	3825	static int nfp_translate(struct nfp_prog *nfp_prog)
3372	3826	{
3373	3827	struct nfp_insn_meta *meta;
	3828	+ unsigned int depth;
3374	3829	int err;
	3830	+
	3831	+ depth = nfp_prog->subprog[0].stack_depth;
	3832	+ nfp_prog->stack_frame_depth = round_up(depth, 4);
3375	3833
3376	3834	nfp_intro(nfp_prog);
3377	3835	if (nfp_prog->error)
..	..	@@ -3382,7 +3840,13 @@
3382	3840
3383	3841	meta->off = nfp_prog_current_offset(nfp_prog);
3384	3842
3385		- if (meta->skip) {
	3843	+ if (nfp_is_subprog_start(meta)) {
	3844	+ nfp_start_subprog(nfp_prog, meta);
	3845	+ if (nfp_prog->error)
	3846	+ return nfp_prog->error;
	3847	+ }
	3848	+
	3849	+ if (meta->flags & FLAG_INSN_SKIP_MASK) {
3386	3850	nfp_prog->n_translated++;
3387	3851	continue;
3388	3852	}
..	..	@@ -3430,10 +3894,10 @@
3430	3894	/* Programs start with R6 = R1 but we ignore the skb pointer */
3431	3895	if (insn.code == (BPF_ALU64 \| BPF_MOV \| BPF_X) &&
3432	3896	insn.src_reg == 1 && insn.dst_reg == 6)
3433		- meta->skip = true;
	3897	+ meta->flags \|= FLAG_INSN_SKIP_PREC_DEPENDENT;
3434	3898
3435	3899	/* Return as soon as something doesn't match */
3436		- if (!meta->skip)
	3900	+ if (!(meta->flags & FLAG_INSN_SKIP_MASK))
3437	3901	return;
3438	3902	}
3439	3903	}
..	..	@@ -3448,19 +3912,17 @@
3448	3912	list_for_each_entry(meta, &nfp_prog->insns, l) {
3449	3913	struct bpf_insn insn = meta->insn;
3450	3914
3451		- if (meta->skip)
	3915	+ if (meta->flags & FLAG_INSN_SKIP_MASK)
3452	3916	continue;
3453	3917
3454		- if (BPF_CLASS(insn.code) != BPF_ALU &&
3455		- BPF_CLASS(insn.code) != BPF_ALU64 &&
3456		- BPF_CLASS(insn.code) != BPF_JMP)
	3918	+ if (!is_mbpf_alu(meta) && !is_mbpf_jmp(meta))
3457	3919	continue;
3458	3920	if (BPF_SRC(insn.code) != BPF_K)
3459	3921	continue;
3460	3922	if (insn.imm >= 0)
3461	3923	continue;
3462	3924
3463		- if (BPF_CLASS(insn.code) == BPF_JMP) {
	3925	+ if (is_mbpf_jmp(meta)) {
3464	3926	switch (BPF_OP(insn.code)) {
3465	3927	case BPF_JGE:
3466	3928	case BPF_JSGE:
..	..	@@ -3490,7 +3952,7 @@
3490	3952	static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog)
3491	3953	{
3492	3954	struct nfp_insn_meta meta1, meta2;
3493		- const s32 exp_mask[] = {
	3955	+ static const s32 exp_mask[] = {
3494	3956	[BPF_B] = 0x000000ffU,
3495	3957	[BPF_H] = 0x0000ffffU,
3496	3958	[BPF_W] = 0xffffffffU,
..	..	@@ -3522,7 +3984,7 @@
3522	3984	if (meta2->flags & FLAG_INSN_IS_JUMP_DST)
3523	3985	continue;
3524	3986
3525		- meta2->skip = true;
	3987	+ meta2->flags \|= FLAG_INSN_SKIP_PREC_DEPENDENT;
3526	3988	}
3527	3989	}
3528	3990
..	..	@@ -3562,8 +4024,8 @@
3562	4024	meta3->flags & FLAG_INSN_IS_JUMP_DST)
3563	4025	continue;
3564	4026
3565		- meta2->skip = true;
3566		- meta3->skip = true;
	4027	+ meta2->flags \|= FLAG_INSN_SKIP_PREC_DEPENDENT;
	4028	+ meta3->flags \|= FLAG_INSN_SKIP_PREC_DEPENDENT;
3567	4029	}
3568	4030	}
3569	4031
..	..	@@ -3758,7 +4220,8 @@
3758	4220	}
3759	4221
3760	4222	head_ld_meta->paired_st = &head_st_meta->insn;
3761		- head_st_meta->skip = true;
	4223	+ head_st_meta->flags \|=
	4224	+ FLAG_INSN_SKIP_PREC_DEPENDENT;
3762	4225	} else {
3763	4226	head_ld_meta->ldst_gather_len = 0;
3764	4227	}
..	..	@@ -3791,8 +4254,8 @@
3791	4254	head_ld_meta = meta1;
3792	4255	head_st_meta = meta2;
3793	4256	} else {
3794		- meta1->skip = true;
3795		- meta2->skip = true;
	4257	+ meta1->flags \|= FLAG_INSN_SKIP_PREC_DEPENDENT;
	4258	+ meta2->flags \|= FLAG_INSN_SKIP_PREC_DEPENDENT;
3796	4259	}
3797	4260
3798	4261	head_ld_meta->ldst_gather_len += BPF_LDST_BYTES(ld);
..	..	@@ -3817,7 +4280,7 @@
3817	4280	if (meta->flags & FLAG_INSN_IS_JUMP_DST)
3818	4281	cache_avail = false;
3819	4282
3820		- if (meta->skip)
	4283	+ if (meta->flags & FLAG_INSN_SKIP_MASK)
3821	4284	continue;
3822	4285
3823	4286	insn = &meta->insn;
..	..	@@ -3903,7 +4366,7 @@
3903	4366	}
3904	4367
3905	4368	list_for_each_entry(meta, &nfp_prog->insns, l) {
3906		- if (meta->skip)
	4369	+ if (meta->flags & FLAG_INSN_SKIP_MASK)
3907	4370	continue;
3908	4371
3909	4372	if (is_mbpf_load_pkt(meta) && !meta->ldst_gather_len) {
..	..	@@ -3939,7 +4402,8 @@
3939	4402	u32 id;
3940	4403
3941	4404	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
3942		- if (meta1->skip \|\| meta2->skip)
	4405	+ if (meta1->flags & FLAG_INSN_SKIP_MASK \|\|
	4406	+ meta2->flags & FLAG_INSN_SKIP_MASK)
3943	4407	continue;
3944	4408
3945	4409	if (meta1->insn.code != (BPF_LD \| BPF_IMM \| BPF_DW) \|\|
..	..	@@ -4018,26 +4482,41 @@
4018	4482	return ret;
4019	4483	}
4020	4484
4021		-void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt)
	4485	+void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog)
4022	4486	{
4023	4487	struct nfp_insn_meta *meta;
4024	4488
4025	4489	/* Another pass to record jump information. */
4026	4490	list_for_each_entry(meta, &nfp_prog->insns, l) {
	4491	+ struct nfp_insn_meta *dst_meta;
4027	4492	u64 code = meta->insn.code;
	4493	+ unsigned int dst_idx;
	4494	+ bool pseudo_call;
4028	4495
4029		- if (BPF_CLASS(code) == BPF_JMP && BPF_OP(code) != BPF_EXIT &&
4030		- BPF_OP(code) != BPF_CALL) {
4031		- struct nfp_insn_meta *dst_meta;
4032		- unsigned short dst_indx;
	4496	+ if (!is_mbpf_jmp(meta))
	4497	+ continue;
	4498	+ if (BPF_OP(code) == BPF_EXIT)
	4499	+ continue;
	4500	+ if (is_mbpf_helper_call(meta))
	4501	+ continue;
4033	4502
4034		- dst_indx = meta->n + 1 + meta->insn.off;
4035		- dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_indx,
4036		- cnt);
	4503	+ /* If opcode is BPF_CALL at this point, this can only be a
	4504	+ * BPF-to-BPF call (a.k.a pseudo call).
	4505	+ */
	4506	+ pseudo_call = BPF_OP(code) == BPF_CALL;
4037	4507
4038		- meta->jmp_dst = dst_meta;
4039		- dst_meta->flags \|= FLAG_INSN_IS_JUMP_DST;
4040		- }
	4508	+ if (pseudo_call)
	4509	+ dst_idx = meta->n + 1 + meta->insn.imm;
	4510	+ else
	4511	+ dst_idx = meta->n + 1 + meta->insn.off;
	4512	+
	4513	+ dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_idx);
	4514	+
	4515	+ if (pseudo_call)
	4516	+ dst_meta->flags \|= FLAG_INSN_IS_SUBPROG_START;
	4517	+
	4518	+ dst_meta->flags \|= FLAG_INSN_IS_JUMP_DST;
	4519	+ meta->jmp_dst = dst_meta;
4041	4520	}
4042	4521	}
4043	4522
..	..	@@ -4060,6 +4539,7 @@
4060	4539	for (i = 0; i < nfp_prog->prog_len; i++) {
4061	4540	enum nfp_relo_type special;
4062	4541	u32 val;
	4542	+ u16 off;
4063	4543
4064	4544	special = FIELD_GET(OP_RELO_TYPE, prog[i]);
4065	4545	switch (special) {
..	..	@@ -4076,6 +4556,24 @@
4076	4556	br_set_offset(&prog[i],
4077	4557	nfp_prog->tgt_abort + bv->start_off);
4078	4558	break;
	4559	+ case RELO_BR_GO_CALL_PUSH_REGS:
	4560	+ if (!nfp_prog->tgt_call_push_regs) {
	4561	+ pr_err("BUG: failed to detect subprogram registers needs\n");
	4562	+ err = -EINVAL;
	4563	+ goto err_free_prog;
	4564	+ }
	4565	+ off = nfp_prog->tgt_call_push_regs + bv->start_off;
	4566	+ br_set_offset(&prog[i], off);
	4567	+ break;
	4568	+ case RELO_BR_GO_CALL_POP_REGS:
	4569	+ if (!nfp_prog->tgt_call_pop_regs) {
	4570	+ pr_err("BUG: failed to detect subprogram registers needs\n");
	4571	+ err = -EINVAL;
	4572	+ goto err_free_prog;
	4573	+ }
	4574	+ off = nfp_prog->tgt_call_pop_regs + bv->start_off;
	4575	+ br_set_offset(&prog[i], off);
	4576	+ break;
4079	4577	case RELO_BR_NEXT_PKT:
4080	4578	br_set_offset(&prog[i], bv->tgt_done);
4081	4579	break;