~hc/RK356X_SDK_RELEASE.git

..	..	@@ -23,6 +23,8 @@
23	23	#include <linux/filter.h>
24	24	#include <linux/init.h>
25	25	#include <linux/bpf.h>
	26	+#include <linux/mm.h>
	27	+#include <linux/kernel.h>
26	28	#include <asm/cacheflush.h>
27	29	#include <asm/dis.h>
28	30	#include <asm/facility.h>
..	..	@@ -38,25 +40,22 @@
38	40	int size; /* Size of program and literal pool */
39	41	int size_prg; /* Size of program */
40	42	int prg; /* Current position in program */
41		- int lit_start; /* Start of literal pool */
42		- int lit; /* Current position in literal pool */
	43	+ int lit32_start; /* Start of 32-bit literal pool */
	44	+ int lit32; /* Current position in 32-bit literal pool */
	45	+ int lit64_start; /* Start of 64-bit literal pool */
	46	+ int lit64; /* Current position in 64-bit literal pool */
43	47	int base_ip; /* Base address for literal pool */
44		- int ret0_ip; /* Address of return 0 */
45	48	int exit_ip; /* Address of exit */
46	49	int r1_thunk_ip; /* Address of expoline thunk for 'br %r1' */
47	50	int r14_thunk_ip; /* Address of expoline thunk for 'br %r14' */
48	51	int tail_call_start; /* Tail call start offset */
49		- int labels[1]; /* Labels for local jumps */
	52	+ int excnt; /* Number of exception table entries */
50	53	};
51	54
52		-#define BPF_SIZE_MAX 0xffff /* Max size for program (16 bit branches) */
53		-
54		-#define SEEN_MEM (1 << 0) /* use mem[] for temporary storage */
55		-#define SEEN_RET0 (1 << 1) /* ret0_ip points to a valid return 0 */
56		-#define SEEN_LITERAL (1 << 2) /* code uses literals */
57		-#define SEEN_FUNC (1 << 3) /* calls C functions */
58		-#define SEEN_TAIL_CALL (1 << 4) /* code uses tail calls */
59		-#define SEEN_REG_AX (1 << 5) /* code uses constant blinding */
	55	+#define SEEN_MEM BIT(0) /* use mem[] for temporary storage */
	56	+#define SEEN_LITERAL BIT(1) /* code uses literals */
	57	+#define SEEN_FUNC BIT(2) /* calls C functions */
	58	+#define SEEN_TAIL_CALL BIT(3) /* code uses tail calls */
60	59	#define SEEN_STACK (SEEN_FUNC \| SEEN_MEM)
61	60
62	61	/*
..	..	@@ -131,13 +130,13 @@
131	130	#define _EMIT2(op) \
132	131	({ \
133	132	if (jit->prg_buf) \
134		- (u16 ) (jit->prg_buf + jit->prg) = op; \
	133	+ (u16 ) (jit->prg_buf + jit->prg) = (op); \
135	134	jit->prg += 2; \
136	135	})
137	136
138	137	#define EMIT2(op, b1, b2) \
139	138	({ \
140		- _EMIT2(op \| reg(b1, b2)); \
	139	+ _EMIT2((op) \| reg(b1, b2)); \
141	140	REG_SET_SEEN(b1); \
142	141	REG_SET_SEEN(b2); \
143	142	})
..	..	@@ -145,20 +144,20 @@
145	144	#define _EMIT4(op) \
146	145	({ \
147	146	if (jit->prg_buf) \
148		- (u32 ) (jit->prg_buf + jit->prg) = op; \
	147	+ (u32 ) (jit->prg_buf + jit->prg) = (op); \
149	148	jit->prg += 4; \
150	149	})
151	150
152	151	#define EMIT4(op, b1, b2) \
153	152	({ \
154		- _EMIT4(op \| reg(b1, b2)); \
	153	+ _EMIT4((op) \| reg(b1, b2)); \
155	154	REG_SET_SEEN(b1); \
156	155	REG_SET_SEEN(b2); \
157	156	})
158	157
159	158	#define EMIT4_RRF(op, b1, b2, b3) \
160	159	({ \
161		- _EMIT4(op \| reg_high(b3) << 8 \| reg(b1, b2)); \
	160	+ _EMIT4((op) \| reg_high(b3) << 8 \| reg(b1, b2)); \
162	161	REG_SET_SEEN(b1); \
163	162	REG_SET_SEEN(b2); \
164	163	REG_SET_SEEN(b3); \
..	..	@@ -167,13 +166,13 @@
167	166	#define _EMIT4_DISP(op, disp) \
168	167	({ \
169	168	unsigned int __disp = (disp) & 0xfff; \
170		- _EMIT4(op \| __disp); \
	169	+ _EMIT4((op) \| __disp); \
171	170	})
172	171
173	172	#define EMIT4_DISP(op, b1, b2, disp) \
174	173	({ \
175		- _EMIT4_DISP(op \| reg_high(b1) << 16 \| \
176		- reg_high(b2) << 8, disp); \
	174	+ _EMIT4_DISP((op) \| reg_high(b1) << 16 \| \
	175	+ reg_high(b2) << 8, (disp)); \
177	176	REG_SET_SEEN(b1); \
178	177	REG_SET_SEEN(b2); \
179	178	})
..	..	@@ -181,21 +180,27 @@
181	180	#define EMIT4_IMM(op, b1, imm) \
182	181	({ \
183	182	unsigned int __imm = (imm) & 0xffff; \
184		- _EMIT4(op \| reg_high(b1) << 16 \| __imm); \
	183	+ _EMIT4((op) \| reg_high(b1) << 16 \| __imm); \
185	184	REG_SET_SEEN(b1); \
186	185	})
187	186
188	187	#define EMIT4_PCREL(op, pcrel) \
189	188	({ \
190	189	long __pcrel = ((pcrel) >> 1) & 0xffff; \
191		- _EMIT4(op \| __pcrel); \
	190	+ _EMIT4((op) \| __pcrel); \
	191	+})
	192	+
	193	+#define EMIT4_PCREL_RIC(op, mask, target) \
	194	+({ \
	195	+ int __rel = ((target) - jit->prg) / 2; \
	196	+ _EMIT4((op) \| (mask) << 20 \| (__rel & 0xffff)); \
192	197	})
193	198
194	199	#define _EMIT6(op1, op2) \
195	200	({ \
196	201	if (jit->prg_buf) { \
197		- (u32 ) (jit->prg_buf + jit->prg) = op1; \
198		- (u16 ) (jit->prg_buf + jit->prg + 4) = op2; \
	202	+ (u32 ) (jit->prg_buf + jit->prg) = (op1); \
	203	+ (u16 ) (jit->prg_buf + jit->prg + 4) = (op2); \
199	204	} \
200	205	jit->prg += 6; \
201	206	})
..	..	@@ -203,106 +208,183 @@
203	208	#define _EMIT6_DISP(op1, op2, disp) \
204	209	({ \
205	210	unsigned int __disp = (disp) & 0xfff; \
206		- _EMIT6(op1 \| __disp, op2); \
	211	+ _EMIT6((op1) \| __disp, op2); \
207	212	})
208	213
209	214	#define _EMIT6_DISP_LH(op1, op2, disp) \
210	215	({ \
211		- u32 _disp = (u32) disp; \
	216	+ u32 _disp = (u32) (disp); \
212	217	unsigned int __disp_h = _disp & 0xff000; \
213	218	unsigned int __disp_l = _disp & 0x00fff; \
214		- _EMIT6(op1 \| __disp_l, op2 \| __disp_h >> 4); \
	219	+ _EMIT6((op1) \| __disp_l, (op2) \| __disp_h >> 4); \
215	220	})
216	221
217	222	#define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp) \
218	223	({ \
219		- _EMIT6_DISP_LH(op1 \| reg(b1, b2) << 16 \| \
	224	+ _EMIT6_DISP_LH((op1) \| reg(b1, b2) << 16 \| \
220	225	reg_high(b3) << 8, op2, disp); \
221	226	REG_SET_SEEN(b1); \
222	227	REG_SET_SEEN(b2); \
223	228	REG_SET_SEEN(b3); \
224	229	})
225	230
226		-#define EMIT6_PCREL_LABEL(op1, op2, b1, b2, label, mask) \
	231	+#define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target) \
227	232	({ \
228		- int rel = (jit->labels[label] - jit->prg) >> 1; \
229		- _EMIT6(op1 \| reg(b1, b2) << 16 \| (rel & 0xffff), \
230		- op2 \| mask << 12); \
	233	+ unsigned int rel = (int)((target) - jit->prg) / 2; \
	234	+ _EMIT6((op1) \| reg(b1, b2) << 16 \| (rel & 0xffff), \
	235	+ (op2) \| (mask) << 12); \
231	236	REG_SET_SEEN(b1); \
232	237	REG_SET_SEEN(b2); \
233	238	})
234	239
235		-#define EMIT6_PCREL_IMM_LABEL(op1, op2, b1, imm, label, mask) \
	240	+#define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target) \
236	241	({ \
237		- int rel = (jit->labels[label] - jit->prg) >> 1; \
238		- _EMIT6(op1 \| (reg_high(b1) \| mask) << 16 \| \
239		- (rel & 0xffff), op2 \| (imm & 0xff) << 8); \
	242	+ unsigned int rel = (int)((target) - jit->prg) / 2; \
	243	+ _EMIT6((op1) \| (reg_high(b1) \| (mask)) << 16 \| \
	244	+ (rel & 0xffff), (op2) \| ((imm) & 0xff) << 8); \
240	245	REG_SET_SEEN(b1); \
241		- BUILD_BUG_ON(((unsigned long) imm) > 0xff); \
	246	+ BUILD_BUG_ON(((unsigned long) (imm)) > 0xff); \
242	247	})
243	248
244	249	#define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \
245	250	({ \
246		- /* Branch instruction needs 6 bytes */ \
247		- int rel = (addrs[i + off + 1] - (addrs[i + 1] - 6)) / 2;\
248		- _EMIT6(op1 \| reg(b1, b2) << 16 \| (rel & 0xffff), op2 \| mask); \
	251	+ int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2; \
	252	+ _EMIT6((op1) \| reg(b1, b2) << 16 \| (rel & 0xffff), (op2) \| (mask));\
249	253	REG_SET_SEEN(b1); \
250	254	REG_SET_SEEN(b2); \
251	255	})
252	256
253	257	#define EMIT6_PCREL_RILB(op, b, target) \
254	258	({ \
255		- int rel = (target - jit->prg) / 2; \
256		- _EMIT6(op \| reg_high(b) << 16 \| rel >> 16, rel & 0xffff); \
	259	+ unsigned int rel = (int)((target) - jit->prg) / 2; \
	260	+ _EMIT6((op) \| reg_high(b) << 16 \| rel >> 16, rel & 0xffff);\
257	261	REG_SET_SEEN(b); \
258	262	})
259	263
260	264	#define EMIT6_PCREL_RIL(op, target) \
261	265	({ \
262		- int rel = (target - jit->prg) / 2; \
263		- _EMIT6(op \| rel >> 16, rel & 0xffff); \
	266	+ unsigned int rel = (int)((target) - jit->prg) / 2; \
	267	+ _EMIT6((op) \| rel >> 16, rel & 0xffff); \
	268	+})
	269	+
	270	+#define EMIT6_PCREL_RILC(op, mask, target) \
	271	+({ \
	272	+ EMIT6_PCREL_RIL((op) \| (mask) << 20, (target)); \
264	273	})
265	274
266	275	#define _EMIT6_IMM(op, imm) \
267	276	({ \
268	277	unsigned int __imm = (imm); \
269		- _EMIT6(op \| (__imm >> 16), __imm & 0xffff); \
	278	+ _EMIT6((op) \| (__imm >> 16), __imm & 0xffff); \
270	279	})
271	280
272	281	#define EMIT6_IMM(op, b1, imm) \
273	282	({ \
274		- _EMIT6_IMM(op \| reg_high(b1) << 16, imm); \
	283	+ _EMIT6_IMM((op) \| reg_high(b1) << 16, imm); \
275	284	REG_SET_SEEN(b1); \
	285	+})
	286	+
	287	+#define _EMIT_CONST_U32(val) \
	288	+({ \
	289	+ unsigned int ret; \
	290	+ ret = jit->lit32; \
	291	+ if (jit->prg_buf) \
	292	+ (u32 )(jit->prg_buf + jit->lit32) = (u32)(val);\
	293	+ jit->lit32 += 4; \
	294	+ ret; \
276	295	})
277	296
278	297	#define EMIT_CONST_U32(val) \
279	298	({ \
280		- unsigned int ret; \
281		- ret = jit->lit - jit->base_ip; \
282	299	jit->seen \|= SEEN_LITERAL; \
	300	+ _EMIT_CONST_U32(val) - jit->base_ip; \
	301	+})
	302	+
	303	+#define _EMIT_CONST_U64(val) \
	304	+({ \
	305	+ unsigned int ret; \
	306	+ ret = jit->lit64; \
283	307	if (jit->prg_buf) \
284		- (u32 ) (jit->prg_buf + jit->lit) = (u32) val; \
285		- jit->lit += 4; \
	308	+ (u64 )(jit->prg_buf + jit->lit64) = (u64)(val);\
	309	+ jit->lit64 += 8; \
286	310	ret; \
287	311	})
288	312
289	313	#define EMIT_CONST_U64(val) \
290	314	({ \
291		- unsigned int ret; \
292		- ret = jit->lit - jit->base_ip; \
293	315	jit->seen \|= SEEN_LITERAL; \
294		- if (jit->prg_buf) \
295		- (u64 ) (jit->prg_buf + jit->lit) = (u64) val; \
296		- jit->lit += 8; \
297		- ret; \
	316	+ _EMIT_CONST_U64(val) - jit->base_ip; \
298	317	})
299	318
300	319	#define EMIT_ZERO(b1) \
301	320	({ \
302		- /* llgfr %dst,%dst (zero extend to 64 bit) */ \
303		- EMIT4(0xb9160000, b1, b1); \
304		- REG_SET_SEEN(b1); \
	321	+ if (!fp->aux->verifier_zext) { \
	322	+ /* llgfr %dst,%dst (zero extend to 64 bit) */ \
	323	+ EMIT4(0xb9160000, b1, b1); \
	324	+ REG_SET_SEEN(b1); \
	325	+ } \
305	326	})
	327	+
	328	+/*
	329	+ * Return whether this is the first pass. The first pass is special, since we
	330	+ * don't know any sizes yet, and thus must be conservative.
	331	+ */
	332	+static bool is_first_pass(struct bpf_jit *jit)
	333	+{
	334	+ return jit->size == 0;
	335	+}
	336	+
	337	+/*
	338	+ * Return whether this is the code generation pass. The code generation pass is
	339	+ * special, since we should change as little as possible.
	340	+ */
	341	+static bool is_codegen_pass(struct bpf_jit *jit)
	342	+{
	343	+ return jit->prg_buf;
	344	+}
	345	+
	346	+/*
	347	+ * Return whether "rel" can be encoded as a short PC-relative offset
	348	+ */
	349	+static bool is_valid_rel(int rel)
	350	+{
	351	+ return rel >= -65536 && rel <= 65534;
	352	+}
	353	+
	354	+/*
	355	+ * Return whether "off" can be reached using a short PC-relative offset
	356	+ */
	357	+static bool can_use_rel(struct bpf_jit *jit, int off)
	358	+{
	359	+ return is_valid_rel(off - jit->prg);
	360	+}
	361	+
	362	+/*
	363	+ * Return whether given displacement can be encoded using
	364	+ * Long-Displacement Facility
	365	+ */
	366	+static bool is_valid_ldisp(int disp)
	367	+{
	368	+ return disp >= -524288 && disp <= 524287;
	369	+}
	370	+
	371	+/*
	372	+ * Return whether the next 32-bit literal pool entry can be referenced using
	373	+ * Long-Displacement Facility
	374	+ */
	375	+static bool can_use_ldisp_for_lit32(struct bpf_jit *jit)
	376	+{
	377	+ return is_valid_ldisp(jit->lit32 - jit->base_ip);
	378	+}
	379	+
	380	+/*
	381	+ * Return whether the next 64-bit literal pool entry can be referenced using
	382	+ * Long-Displacement Facility
	383	+ */
	384	+static bool can_use_ldisp_for_lit64(struct bpf_jit *jit)
	385	+{
	386	+ return is_valid_ldisp(jit->lit64 - jit->base_ip);
	387	+}
306	388
307	389	/*
308	390	* Fill whole space with illegal instructions
..	..	@@ -381,8 +463,17 @@
381	463	*/
382	464	static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth)
383	465	{
384		-
	466	+ const int last = 15, save_restore_size = 6;
385	467	int re = 6, rs;
	468	+
	469	+ if (is_first_pass(jit)) {
	470	+ /*
	471	+ * We don't know yet which registers are used. Reserve space
	472	+ * conservatively.
	473	+ */
	474	+ jit->prg += (last - re + 1) * save_restore_size;
	475	+ return;
	476	+ }
386	477
387	478	do {
388	479	rs = get_start(jit, re);
..	..	@@ -394,7 +485,25 @@
394	485	else
395	486	restore_regs(jit, rs, re, stack_depth);
396	487	re++;
397		- } while (re <= 15);
	488	+ } while (re <= last);
	489	+}
	490	+
	491	+static void bpf_skip(struct bpf_jit *jit, int size)
	492	+{
	493	+ if (size >= 6 && !is_valid_rel(size)) {
	494	+ /* brcl 0xf,size */
	495	+ EMIT6_PCREL_RIL(0xc0f4000000, size);
	496	+ size -= 6;
	497	+ } else if (size >= 4 && is_valid_rel(size)) {
	498	+ /* brc 0xf,size */
	499	+ EMIT4_PCREL(0xa7f40000, size);
	500	+ size -= 4;
	501	+ }
	502	+ while (size >= 2) {
	503	+ /* bcr 0,%0 */
	504	+ _EMIT2(0x0700);
	505	+ size -= 2;
	506	+ }
398	507	}
399	508
400	509	/*
..	..	@@ -409,30 +518,39 @@
409	518	/* xc STK_OFF_TCCNT(4,%r15),STK_OFF_TCCNT(%r15) */
410	519	_EMIT6(0xd703f000 \| STK_OFF_TCCNT, 0xf000 \| STK_OFF_TCCNT);
411	520	} else {
412		- /* j tail_call_start: NOP if no tail calls are used */
413		- EMIT4_PCREL(0xa7f40000, 6);
414		- _EMIT2(0);
	521	+ /*
	522	+ * There are no tail calls. Insert nops in order to have
	523	+ * tail_call_start at a predictable offset.
	524	+ */
	525	+ bpf_skip(jit, 6);
415	526	}
416	527	/* Tail calls have to skip above initialization */
417	528	jit->tail_call_start = jit->prg;
418	529	/* Save registers */
419	530	save_restore_regs(jit, REGS_SAVE, stack_depth);
420	531	/* Setup literal pool */
421		- if (jit->seen & SEEN_LITERAL) {
422		- /* basr %r13,0 */
423		- EMIT2(0x0d00, REG_L, REG_0);
424		- jit->base_ip = jit->prg;
	532	+ if (is_first_pass(jit) \|\| (jit->seen & SEEN_LITERAL)) {
	533	+ if (!is_first_pass(jit) &&
	534	+ is_valid_ldisp(jit->size - (jit->prg + 2))) {
	535	+ /* basr %l,0 */
	536	+ EMIT2(0x0d00, REG_L, REG_0);
	537	+ jit->base_ip = jit->prg;
	538	+ } else {
	539	+ /* larl %l,lit32_start */
	540	+ EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start);
	541	+ jit->base_ip = jit->lit32_start;
	542	+ }
425	543	}
426	544	/* Setup stack and backchain */
427		- if (jit->seen & SEEN_STACK) {
428		- if (jit->seen & SEEN_FUNC)
	545	+ if (is_first_pass(jit) \|\| (jit->seen & SEEN_STACK)) {
	546	+ if (is_first_pass(jit) \|\| (jit->seen & SEEN_FUNC))
429	547	/* lgr %w1,%r15 (backchain) */
430	548	EMIT4(0xb9040000, REG_W1, REG_15);
431	549	/* la %bfp,STK_160_UNUSED(%r15) (BPF frame pointer) */
432	550	EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, STK_160_UNUSED);
433	551	/* aghi %r15,-STK_OFF */
434	552	EMIT4_IMM(0xa70b0000, REG_15, -(STK_OFF + stack_depth));
435		- if (jit->seen & SEEN_FUNC)
	553	+ if (is_first_pass(jit) \|\| (jit->seen & SEEN_FUNC))
436	554	/* stg %w1,152(%r15) (backchain) */
437	555	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
438	556	REG_15, 152);
..	..	@@ -444,18 +562,12 @@
444	562	*/
445	563	static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
446	564	{
447		- /* Return 0 */
448		- if (jit->seen & SEEN_RET0) {
449		- jit->ret0_ip = jit->prg;
450		- /* lghi %b0,0 */
451		- EMIT4_IMM(0xa7090000, BPF_REG_0, 0);
452		- }
453	565	jit->exit_ip = jit->prg;
454	566	/* Load exit code: lgr %r2,%b0 */
455	567	EMIT4(0xb9040000, REG_2, BPF_REG_0);
456	568	/* Restore registers */
457	569	save_restore_regs(jit, REGS_RESTORE, stack_depth);
458		- if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) {
	570	+ if (__is_defined(CC_USING_EXPOLINE) && !nospec_disable) {
459	571	jit->r14_thunk_ip = jit->prg;
460	572	/* Generate __s390_indirect_jump_r14 thunk */
461	573	if (test_facility(35)) {
..	..	@@ -473,8 +585,8 @@
473	585	/* br %r14 */
474	586	_EMIT2(0x07fe);
475	587
476		- if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable &&
477		- (jit->seen & SEEN_FUNC)) {
	588	+ if (__is_defined(CC_USING_EXPOLINE) && !nospec_disable &&
	589	+ (is_first_pass(jit) \|\| (jit->seen & SEEN_FUNC))) {
478	590	jit->r1_thunk_ip = jit->prg;
479	591	/* Generate __s390_indirect_jump_r1 thunk */
480	592	if (test_facility(35)) {
..	..	@@ -494,25 +606,109 @@
494	606	}
495	607	}
496	608
	609	+static int get_probe_mem_regno(const u8 *insn)
	610	+{
	611	+ /*
	612	+ * insn must point to llgc, llgh, llgf or lg, which have destination
	613	+ * register at the same position.
	614	+ */
	615	+ if (insn[0] != 0xe3) /* common llgc, llgh, llgf and lg prefix */
	616	+ return -1;
	617	+ if (insn[5] != 0x90 && /* llgc */
	618	+ insn[5] != 0x91 && /* llgh */
	619	+ insn[5] != 0x16 && /* llgf */
	620	+ insn[5] != 0x04) /* lg */
	621	+ return -1;
	622	+ return insn[1] >> 4;
	623	+}
	624	+
	625	+static bool ex_handler_bpf(const struct exception_table_entry *x,
	626	+ struct pt_regs *regs)
	627	+{
	628	+ int regno;
	629	+ u8 *insn;
	630	+
	631	+ regs->psw.addr = extable_fixup(x);
	632	+ insn = (u8 *)__rewind_psw(regs->psw, regs->int_code >> 16);
	633	+ regno = get_probe_mem_regno(insn);
	634	+ if (WARN_ON_ONCE(regno < 0))
	635	+ /* JIT bug - unexpected instruction. */
	636	+ return false;
	637	+ regs->gprs[regno] = 0;
	638	+ return true;
	639	+}
	640	+
	641	+static int bpf_jit_probe_mem(struct bpf_jit jit, struct bpf_prog fp,
	642	+ int probe_prg, int nop_prg)
	643	+{
	644	+ struct exception_table_entry *ex;
	645	+ s64 delta;
	646	+ u8 *insn;
	647	+ int prg;
	648	+ int i;
	649	+
	650	+ if (!fp->aux->extable)
	651	+ /* Do nothing during early JIT passes. */
	652	+ return 0;
	653	+ insn = jit->prg_buf + probe_prg;
	654	+ if (WARN_ON_ONCE(get_probe_mem_regno(insn) < 0))
	655	+ /* JIT bug - unexpected probe instruction. */
	656	+ return -1;
	657	+ if (WARN_ON_ONCE(probe_prg + insn_length(*insn) != nop_prg))
	658	+ /* JIT bug - gap between probe and nop instructions. */
	659	+ return -1;
	660	+ for (i = 0; i < 2; i++) {
	661	+ if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries))
	662	+ /* Verifier bug - not enough entries. */
	663	+ return -1;
	664	+ ex = &fp->aux->extable[jit->excnt];
	665	+ /* Add extable entries for probe and nop instructions. */
	666	+ prg = i == 0 ? probe_prg : nop_prg;
	667	+ delta = jit->prg_buf + prg - (u8 *)&ex->insn;
	668	+ if (WARN_ON_ONCE(delta < INT_MIN \|\| delta > INT_MAX))
	669	+ /* JIT bug - code and extable must be close. */
	670	+ return -1;
	671	+ ex->insn = delta;
	672	+ /*
	673	+ * Always land on the nop. Note that extable infrastructure
	674	+ * ignores fixup field, it is handled by ex_handler_bpf().
	675	+ */
	676	+ delta = jit->prg_buf + nop_prg - (u8 *)&ex->fixup;
	677	+ if (WARN_ON_ONCE(delta < INT_MIN \|\| delta > INT_MAX))
	678	+ /* JIT bug - landing pad and extable must be close. */
	679	+ return -1;
	680	+ ex->fixup = delta;
	681	+ ex->handler = (u8 )ex_handler_bpf - (u8 )&ex->handler;
	682	+ jit->excnt++;
	683	+ }
	684	+ return 0;
	685	+}
	686	+
497	687	/*
498	688	* Compile one eBPF instruction into s390x code
499	689	*
500	690	* NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of
501	691	* stack space for the large switch statement.
502	692	*/
503		-static noinline int bpf_jit_insn(struct bpf_jit jit, struct bpf_prog fp, int i)
	693	+static noinline int bpf_jit_insn(struct bpf_jit jit, struct bpf_prog fp,
	694	+ int i, bool extra_pass, u32 stack_depth)
504	695	{
505	696	struct bpf_insn *insn = &fp->insnsi[i];
506		- int jmp_off, last, insn_count = 1;
507	697	u32 dst_reg = insn->dst_reg;
508	698	u32 src_reg = insn->src_reg;
	699	+ int last, insn_count = 1;
509	700	u32 *addrs = jit->addrs;
510	701	s32 imm = insn->imm;
511	702	s16 off = insn->off;
	703	+ int probe_prg = -1;
512	704	unsigned int mask;
	705	+ int nop_prg;
	706	+ int err;
513	707
514		- if (dst_reg == BPF_REG_AX \|\| src_reg == BPF_REG_AX)
515		- jit->seen \|= SEEN_REG_AX;
	708	+ if (BPF_CLASS(insn->code) == BPF_LDX &&
	709	+ BPF_MODE(insn->code) == BPF_PROBE_MEM)
	710	+ probe_prg = jit->prg;
	711	+
516	712	switch (insn->code) {
517	713	/*
518	714	* BPF_MOV
..	..	@@ -520,6 +716,8 @@
520	716	case BPF_ALU \| BPF_MOV \| BPF_X: /* dst = (u32) src */
521	717	/* llgfr %dst,%src */
522	718	EMIT4(0xb9160000, dst_reg, src_reg);
	719	+ if (insn_is_zext(&insn[1]))
	720	+ insn_count = 2;
523	721	break;
524	722	case BPF_ALU64 \| BPF_MOV \| BPF_X: /* dst = src */
525	723	/* lgr %dst,%src */
..	..	@@ -528,6 +726,8 @@
528	726	case BPF_ALU \| BPF_MOV \| BPF_K: /* dst = (u32) imm */
529	727	/* llilf %dst,imm */
530	728	EMIT6_IMM(0xc00f0000, dst_reg, imm);
	729	+ if (insn_is_zext(&insn[1]))
	730	+ insn_count = 2;
531	731	break;
532	732	case BPF_ALU64 \| BPF_MOV \| BPF_K: /* dst = imm */
533	733	/* lgfi %dst,imm */
..	..	@@ -542,9 +742,8 @@
542	742	u64 imm64;
543	743
544	744	imm64 = (u64)(u32) insn[0].imm \| ((u64)(u32) insn[1].imm) << 32;
545		- /* lg %dst,<d(imm)>(%l) */
546		- EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, REG_0, REG_L,
547		- EMIT_CONST_U64(imm64));
	745	+ /* lgrl %dst,imm */
	746	+ EMIT6_PCREL_RILB(0xc4080000, dst_reg, _EMIT_CONST_U64(imm64));
548	747	insn_count = 2;
549	748	break;
550	749	}
..	..	@@ -644,6 +843,8 @@
644	843	EMIT4(0xb9970000, REG_W0, src_reg);
645	844	/* llgfr %dst,%rc */
646	845	EMIT4(0xb9160000, dst_reg, rc_reg);
	846	+ if (insn_is_zext(&insn[1]))
	847	+ insn_count = 2;
647	848	break;
648	849	}
649	850	case BPF_ALU64 \| BPF_DIV \| BPF_X: /* dst = dst / src */
..	..	@@ -678,11 +879,22 @@
678	879	EMIT4_IMM(0xa7080000, REG_W0, 0);
679	880	/* lr %w1,%dst */
680	881	EMIT2(0x1800, REG_W1, dst_reg);
681		- /* dl %w0,<d(imm)>(%l) */
682		- EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, REG_L,
683		- EMIT_CONST_U32(imm));
	882	+ if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) {
	883	+ /* dl %w0,<d(imm)>(%l) */
	884	+ EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, REG_L,
	885	+ EMIT_CONST_U32(imm));
	886	+ } else {
	887	+ /* lgfrl %dst,imm */
	888	+ EMIT6_PCREL_RILB(0xc40c0000, dst_reg,
	889	+ _EMIT_CONST_U32(imm));
	890	+ jit->seen \|= SEEN_LITERAL;
	891	+ /* dlr %w0,%dst */
	892	+ EMIT4(0xb9970000, REG_W0, dst_reg);
	893	+ }
684	894	/* llgfr %dst,%rc */
685	895	EMIT4(0xb9160000, dst_reg, rc_reg);
	896	+ if (insn_is_zext(&insn[1]))
	897	+ insn_count = 2;
686	898	break;
687	899	}
688	900	case BPF_ALU64 \| BPF_DIV \| BPF_K: /* dst = dst / imm */
..	..	@@ -700,9 +912,18 @@
700	912	EMIT4_IMM(0xa7090000, REG_W0, 0);
701	913	/* lgr %w1,%dst */
702	914	EMIT4(0xb9040000, REG_W1, dst_reg);
703		- /* dlg %w0,<d(imm)>(%l) */
704		- EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, REG_L,
705		- EMIT_CONST_U64(imm));
	915	+ if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
	916	+ /* dlg %w0,<d(imm)>(%l) */
	917	+ EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, REG_L,
	918	+ EMIT_CONST_U64(imm));
	919	+ } else {
	920	+ /* lgrl %dst,imm */
	921	+ EMIT6_PCREL_RILB(0xc4080000, dst_reg,
	922	+ _EMIT_CONST_U64(imm));
	923	+ jit->seen \|= SEEN_LITERAL;
	924	+ /* dlgr %w0,%dst */
	925	+ EMIT4(0xb9870000, REG_W0, dst_reg);
	926	+ }
706	927	/* lgr %dst,%rc */
707	928	EMIT4(0xb9040000, dst_reg, rc_reg);
708	929	break;
..	..	@@ -725,9 +946,19 @@
725	946	EMIT_ZERO(dst_reg);
726	947	break;
727	948	case BPF_ALU64 \| BPF_AND \| BPF_K: /* dst = dst & imm */
728		- /* ng %dst,<d(imm)>(%l) */
729		- EMIT6_DISP_LH(0xe3000000, 0x0080, dst_reg, REG_0, REG_L,
730		- EMIT_CONST_U64(imm));
	949	+ if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
	950	+ /* ng %dst,<d(imm)>(%l) */
	951	+ EMIT6_DISP_LH(0xe3000000, 0x0080,
	952	+ dst_reg, REG_0, REG_L,
	953	+ EMIT_CONST_U64(imm));
	954	+ } else {
	955	+ /* lgrl %w0,imm */
	956	+ EMIT6_PCREL_RILB(0xc4080000, REG_W0,
	957	+ _EMIT_CONST_U64(imm));
	958	+ jit->seen \|= SEEN_LITERAL;
	959	+ /* ngr %dst,%w0 */
	960	+ EMIT4(0xb9800000, dst_reg, REG_W0);
	961	+ }
731	962	break;
732	963	/*
733	964	* BPF_OR
..	..	@@ -747,9 +978,19 @@
747	978	EMIT_ZERO(dst_reg);
748	979	break;
749	980	case BPF_ALU64 \| BPF_OR \| BPF_K: /* dst = dst \| imm */
750		- /* og %dst,<d(imm)>(%l) */
751		- EMIT6_DISP_LH(0xe3000000, 0x0081, dst_reg, REG_0, REG_L,
752		- EMIT_CONST_U64(imm));
	981	+ if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
	982	+ /* og %dst,<d(imm)>(%l) */
	983	+ EMIT6_DISP_LH(0xe3000000, 0x0081,
	984	+ dst_reg, REG_0, REG_L,
	985	+ EMIT_CONST_U64(imm));
	986	+ } else {
	987	+ /* lgrl %w0,imm */
	988	+ EMIT6_PCREL_RILB(0xc4080000, REG_W0,
	989	+ _EMIT_CONST_U64(imm));
	990	+ jit->seen \|= SEEN_LITERAL;
	991	+ /* ogr %dst,%w0 */
	992	+ EMIT4(0xb9810000, dst_reg, REG_W0);
	993	+ }
753	994	break;
754	995	/*
755	996	* BPF_XOR
..	..	@@ -771,9 +1012,19 @@
771	1012	EMIT_ZERO(dst_reg);
772	1013	break;
773	1014	case BPF_ALU64 \| BPF_XOR \| BPF_K: /* dst = dst ^ imm */
774		- /* xg %dst,<d(imm)>(%l) */
775		- EMIT6_DISP_LH(0xe3000000, 0x0082, dst_reg, REG_0, REG_L,
776		- EMIT_CONST_U64(imm));
	1015	+ if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
	1016	+ /* xg %dst,<d(imm)>(%l) */
	1017	+ EMIT6_DISP_LH(0xe3000000, 0x0082,
	1018	+ dst_reg, REG_0, REG_L,
	1019	+ EMIT_CONST_U64(imm));
	1020	+ } else {
	1021	+ /* lgrl %w0,imm */
	1022	+ EMIT6_PCREL_RILB(0xc4080000, REG_W0,
	1023	+ _EMIT_CONST_U64(imm));
	1024	+ jit->seen \|= SEEN_LITERAL;
	1025	+ /* xgr %dst,%w0 */
	1026	+ EMIT4(0xb9820000, dst_reg, REG_W0);
	1027	+ }
777	1028	break;
778	1029	/*
779	1030	* BPF_LSH
..	..	@@ -828,9 +1079,21 @@
828	1079	/*
829	1080	* BPF_ARSH
830	1081	*/
	1082	+ case BPF_ALU \| BPF_ARSH \| BPF_X: /* ((s32) dst) >>= src */
	1083	+ /* sra %dst,%dst,0(%src) */
	1084	+ EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0);
	1085	+ EMIT_ZERO(dst_reg);
	1086	+ break;
831	1087	case BPF_ALU64 \| BPF_ARSH \| BPF_X: /* ((s64) dst) >>= src */
832	1088	/* srag %dst,%dst,0(%src) */
833	1089	EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0);
	1090	+ break;
	1091	+ case BPF_ALU \| BPF_ARSH \| BPF_K: /* ((s32) dst >> imm */
	1092	+ if (imm != 0) {
	1093	+ /* sra %dst,imm(%r0) */
	1094	+ EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm);
	1095	+ }
	1096	+ EMIT_ZERO(dst_reg);
834	1097	break;
835	1098	case BPF_ALU64 \| BPF_ARSH \| BPF_K: /* ((s64) dst) >>= imm */
836	1099	if (imm == 0)
..	..	@@ -859,10 +1122,13 @@
859	1122	case 16: /* dst = (u16) cpu_to_be16(dst) */
860	1123	/* llghr %dst,%dst */
861	1124	EMIT4(0xb9850000, dst_reg, dst_reg);
	1125	+ if (insn_is_zext(&insn[1]))
	1126	+ insn_count = 2;
862	1127	break;
863	1128	case 32: /* dst = (u32) cpu_to_be32(dst) */
864		- /* llgfr %dst,%dst */
865		- EMIT4(0xb9160000, dst_reg, dst_reg);
	1129	+ if (!fp->aux->verifier_zext)
	1130	+ /* llgfr %dst,%dst */
	1131	+ EMIT4(0xb9160000, dst_reg, dst_reg);
866	1132	break;
867	1133	case 64: /* dst = (u64) cpu_to_be64(dst) */
868	1134	break;
..	..	@@ -877,12 +1143,15 @@
877	1143	EMIT4_DISP(0x88000000, dst_reg, REG_0, 16);
878	1144	/* llghr %dst,%dst */
879	1145	EMIT4(0xb9850000, dst_reg, dst_reg);
	1146	+ if (insn_is_zext(&insn[1]))
	1147	+ insn_count = 2;
880	1148	break;
881	1149	case 32: /* dst = (u32) cpu_to_le32(dst) */
882	1150	/* lrvr %dst,%dst */
883	1151	EMIT4(0xb91f0000, dst_reg, dst_reg);
884		- /* llgfr %dst,%dst */
885		- EMIT4(0xb9160000, dst_reg, dst_reg);
	1152	+ if (!fp->aux->verifier_zext)
	1153	+ /* llgfr %dst,%dst */
	1154	+ EMIT4(0xb9160000, dst_reg, dst_reg);
886	1155	break;
887	1156	case 64: /* dst = (u64) cpu_to_le64(dst) */
888	1157	/* lrvgr %dst,%dst */
..	..	@@ -965,21 +1234,31 @@
965	1234	* BPF_LDX
966	1235	*/
967	1236	case BPF_LDX \| BPF_MEM \| BPF_B: /* dst = (u8 )(ul) (src + off) */
	1237	+ case BPF_LDX \| BPF_PROBE_MEM \| BPF_B:
968	1238	/* llgc %dst,0(off,%src) */
969	1239	EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, REG_0, off);
970	1240	jit->seen \|= SEEN_MEM;
	1241	+ if (insn_is_zext(&insn[1]))
	1242	+ insn_count = 2;
971	1243	break;
972	1244	case BPF_LDX \| BPF_MEM \| BPF_H: /* dst = (u16 )(ul) (src + off) */
	1245	+ case BPF_LDX \| BPF_PROBE_MEM \| BPF_H:
973	1246	/* llgh %dst,0(off,%src) */
974	1247	EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, REG_0, off);
975	1248	jit->seen \|= SEEN_MEM;
	1249	+ if (insn_is_zext(&insn[1]))
	1250	+ insn_count = 2;
976	1251	break;
977	1252	case BPF_LDX \| BPF_MEM \| BPF_W: /* dst = (u32 )(ul) (src + off) */
	1253	+ case BPF_LDX \| BPF_PROBE_MEM \| BPF_W:
978	1254	/* llgf %dst,off(%src) */
979	1255	jit->seen \|= SEEN_MEM;
980	1256	EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, REG_0, off);
	1257	+ if (insn_is_zext(&insn[1]))
	1258	+ insn_count = 2;
981	1259	break;
982	1260	case BPF_LDX \| BPF_MEM \| BPF_DW: /* dst = (u64 )(ul) (src + off) */
	1261	+ case BPF_LDX \| BPF_PROBE_MEM \| BPF_DW:
983	1262	/* lg %dst,0(off,%src) */
984	1263	jit->seen \|= SEEN_MEM;
985	1264	EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, REG_0, off);
..	..	@@ -989,17 +1268,20 @@
989	1268	*/
990	1269	case BPF_JMP \| BPF_CALL:
991	1270	{
992		- /*
993		- * b0 = (__bpf_call_base + imm)(b1, b2, b3, b4, b5)
994		- */
995		- const u64 func = (u64)__bpf_call_base + imm;
	1271	+ u64 func;
	1272	+ bool func_addr_fixed;
	1273	+ int ret;
	1274	+
	1275	+ ret = bpf_jit_get_func_addr(fp, insn, extra_pass,
	1276	+ &func, &func_addr_fixed);
	1277	+ if (ret < 0)
	1278	+ return -1;
996	1279
997	1280	REG_SET_SEEN(BPF_REG_5);
998	1281	jit->seen \|= SEEN_FUNC;
999		- /* lg %w1,<d(imm)>(%l) */
1000		- EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W1, REG_0, REG_L,
1001		- EMIT_CONST_U64(func));
1002		- if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) {
	1282	+ /* lgrl %w1,func */
	1283	+ EMIT6_PCREL_RILB(0xc4080000, REG_W1, _EMIT_CONST_U64(func));
	1284	+ if (__is_defined(CC_USING_EXPOLINE) && !nospec_disable) {
1003	1285	/* brasl %r14,__s390_indirect_jump_r1 */
1004	1286	EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);
1005	1287	} else {
..	..	@@ -1010,7 +1292,9 @@
1010	1292	EMIT4(0xb9040000, BPF_REG_0, REG_2);
1011	1293	break;
1012	1294	}
1013		- case BPF_JMP \| BPF_TAIL_CALL:
	1295	+ case BPF_JMP \| BPF_TAIL_CALL: {
	1296	+ int patch_1_clrj, patch_2_clij, patch_3_brc;
	1297	+
1014	1298	/*
1015	1299	* Implicit input:
1016	1300	* B1: pointer to ctx
..	..	@@ -1027,9 +1311,11 @@
1027	1311	/* llgf %w1,map.max_entries(%b2) */
1028	1312	EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_2,
1029	1313	offsetof(struct bpf_array, map.max_entries));
1030		- /* clrj %b3,%w1,0xa,label0: if (u32)%b3 >= (u32)%w1 goto out */
1031		- EMIT6_PCREL_LABEL(0xec000000, 0x0077, BPF_REG_3,
1032		- REG_W1, 0, 0xa);
	1314	+ /* if ((u32)%b3 >= (u32)%w1) goto out; */
	1315	+ /* clrj %b3,%w1,0xa,out */
	1316	+ patch_1_clrj = jit->prg;
	1317	+ EMIT6_PCREL_RIEB(0xec000000, 0x0077, BPF_REG_3, REG_W1, 0xa,
	1318	+ jit->prg);
1033	1319
1034	1320	/*
1035	1321	* if (tail_call_cnt++ > MAX_TAIL_CALL_CNT)
..	..	@@ -1037,16 +1323,17 @@
1037	1323	*/
1038	1324
1039	1325	if (jit->seen & SEEN_STACK)
1040		- off = STK_OFF_TCCNT + STK_OFF + fp->aux->stack_depth;
	1326	+ off = STK_OFF_TCCNT + STK_OFF + stack_depth;
1041	1327	else
1042	1328	off = STK_OFF_TCCNT;
1043	1329	/* lhi %w0,1 */
1044	1330	EMIT4_IMM(0xa7080000, REG_W0, 1);
1045	1331	/* laal %w1,%w0,off(%r15) */
1046	1332	EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off);
1047		- /* clij %w1,MAX_TAIL_CALL_CNT,0x2,label0 */
1048		- EMIT6_PCREL_IMM_LABEL(0xec000000, 0x007f, REG_W1,
1049		- MAX_TAIL_CALL_CNT, 0, 0x2);
	1333	+ /* clij %w1,MAX_TAIL_CALL_CNT,0x2,out */
	1334	+ patch_2_clij = jit->prg;
	1335	+ EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W1, MAX_TAIL_CALL_CNT,
	1336	+ 2, jit->prg);
1050	1337
1051	1338	/*
1052	1339	* prog = array->ptrs[index];
..	..	@@ -1058,16 +1345,17 @@
1058	1345	EMIT4(0xb9160000, REG_1, BPF_REG_3);
1059	1346	/* sllg %r1,%r1,3: %r1 = 8 /
1060	1347	EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, REG_1, REG_0, 3);
1061		- /* lg %r1,prog(%b2,%r1) */
1062		- EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, BPF_REG_2,
	1348	+ /* ltg %r1,prog(%b2,%r1) */
	1349	+ EMIT6_DISP_LH(0xe3000000, 0x0002, REG_1, BPF_REG_2,
1063	1350	REG_1, offsetof(struct bpf_array, ptrs));
1064		- /* clgij %r1,0,0x8,label0 */
1065		- EMIT6_PCREL_IMM_LABEL(0xec000000, 0x007d, REG_1, 0, 0, 0x8);
	1351	+ /* brc 0x8,out */
	1352	+ patch_3_brc = jit->prg;
	1353	+ EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg);
1066	1354
1067	1355	/*
1068	1356	* Restore registers before calling function
1069	1357	*/
1070		- save_restore_regs(jit, REGS_RESTORE, fp->aux->stack_depth);
	1358	+ save_restore_regs(jit, REGS_RESTORE, stack_depth);
1071	1359
1072	1360	/*
1073	1361	* goto *(prog->bpf_func + tail_call_start);
..	..	@@ -1079,14 +1367,26 @@
1079	1367	/* bc 0xf,tail_call_start(%r1) */
1080	1368	_EMIT4(0x47f01000 + jit->tail_call_start);
1081	1369	/* out: */
1082		- jit->labels[0] = jit->prg;
	1370	+ if (jit->prg_buf) {
	1371	+ (u16 )(jit->prg_buf + patch_1_clrj + 2) =
	1372	+ (jit->prg - patch_1_clrj) >> 1;
	1373	+ (u16 )(jit->prg_buf + patch_2_clij + 2) =
	1374	+ (jit->prg - patch_2_clij) >> 1;
	1375	+ (u16 )(jit->prg_buf + patch_3_brc + 2) =
	1376	+ (jit->prg - patch_3_brc) >> 1;
	1377	+ }
1083	1378	break;
	1379	+ }
1084	1380	case BPF_JMP \| BPF_EXIT: /* return b0 */
1085	1381	last = (i == fp->len - 1) ? 1 : 0;
1086		- if (last && !(jit->seen & SEEN_RET0))
	1382	+ if (last)
1087	1383	break;
1088		- /* j <exit> */
1089		- EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);
	1384	+ if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip))
	1385	+ /* brc 0xf, <exit> */
	1386	+ EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip);
	1387	+ else
	1388	+ /* brcl 0xf, <exit> */
	1389	+ EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip);
1090	1390	break;
1091	1391	/*
1092	1392	* Branch relative (number of skipped instructions) to offset on
..	..	@@ -1112,134 +1412,317 @@
1112	1412	mask = 0xf000; /* j */
1113	1413	goto branch_oc;
1114	1414	case BPF_JMP \| BPF_JSGT \| BPF_K: /* ((s64) dst > (s64) imm) */
	1415	+ case BPF_JMP32 \| BPF_JSGT \| BPF_K: /* ((s32) dst > (s32) imm) */
1115	1416	mask = 0x2000; /* jh */
1116	1417	goto branch_ks;
1117	1418	case BPF_JMP \| BPF_JSLT \| BPF_K: /* ((s64) dst < (s64) imm) */
	1419	+ case BPF_JMP32 \| BPF_JSLT \| BPF_K: /* ((s32) dst < (s32) imm) */
1118	1420	mask = 0x4000; /* jl */
1119	1421	goto branch_ks;
1120	1422	case BPF_JMP \| BPF_JSGE \| BPF_K: /* ((s64) dst >= (s64) imm) */
	1423	+ case BPF_JMP32 \| BPF_JSGE \| BPF_K: /* ((s32) dst >= (s32) imm) */
1121	1424	mask = 0xa000; /* jhe */
1122	1425	goto branch_ks;
1123	1426	case BPF_JMP \| BPF_JSLE \| BPF_K: /* ((s64) dst <= (s64) imm) */
	1427	+ case BPF_JMP32 \| BPF_JSLE \| BPF_K: /* ((s32) dst <= (s32) imm) */
1124	1428	mask = 0xc000; /* jle */
1125	1429	goto branch_ks;
1126	1430	case BPF_JMP \| BPF_JGT \| BPF_K: /* (dst_reg > imm) */
	1431	+ case BPF_JMP32 \| BPF_JGT \| BPF_K: /* ((u32) dst_reg > (u32) imm) */
1127	1432	mask = 0x2000; /* jh */
1128	1433	goto branch_ku;
1129	1434	case BPF_JMP \| BPF_JLT \| BPF_K: /* (dst_reg < imm) */
	1435	+ case BPF_JMP32 \| BPF_JLT \| BPF_K: /* ((u32) dst_reg < (u32) imm) */
1130	1436	mask = 0x4000; /* jl */
1131	1437	goto branch_ku;
1132	1438	case BPF_JMP \| BPF_JGE \| BPF_K: /* (dst_reg >= imm) */
	1439	+ case BPF_JMP32 \| BPF_JGE \| BPF_K: /* ((u32) dst_reg >= (u32) imm) */
1133	1440	mask = 0xa000; /* jhe */
1134	1441	goto branch_ku;
1135	1442	case BPF_JMP \| BPF_JLE \| BPF_K: /* (dst_reg <= imm) */
	1443	+ case BPF_JMP32 \| BPF_JLE \| BPF_K: /* ((u32) dst_reg <= (u32) imm) */
1136	1444	mask = 0xc000; /* jle */
1137	1445	goto branch_ku;
1138	1446	case BPF_JMP \| BPF_JNE \| BPF_K: /* (dst_reg != imm) */
	1447	+ case BPF_JMP32 \| BPF_JNE \| BPF_K: /* ((u32) dst_reg != (u32) imm) */
1139	1448	mask = 0x7000; /* jne */
1140	1449	goto branch_ku;
1141	1450	case BPF_JMP \| BPF_JEQ \| BPF_K: /* (dst_reg == imm) */
	1451	+ case BPF_JMP32 \| BPF_JEQ \| BPF_K: /* ((u32) dst_reg == (u32) imm) */
1142	1452	mask = 0x8000; /* je */
1143	1453	goto branch_ku;
1144	1454	case BPF_JMP \| BPF_JSET \| BPF_K: /* (dst_reg & imm) */
	1455	+ case BPF_JMP32 \| BPF_JSET \| BPF_K: /* ((u32) dst_reg & (u32) imm) */
1145	1456	mask = 0x7000; /* jnz */
1146		- /* lgfi %w1,imm (load sign extend imm) */
1147		- EMIT6_IMM(0xc0010000, REG_W1, imm);
1148		- /* ngr %w1,%dst */
1149		- EMIT4(0xb9800000, REG_W1, dst_reg);
	1457	+ if (BPF_CLASS(insn->code) == BPF_JMP32) {
	1458	+ /* llilf %w1,imm (load zero extend imm) */
	1459	+ EMIT6_IMM(0xc00f0000, REG_W1, imm);
	1460	+ /* nr %w1,%dst */
	1461	+ EMIT2(0x1400, REG_W1, dst_reg);
	1462	+ } else {
	1463	+ /* lgfi %w1,imm (load sign extend imm) */
	1464	+ EMIT6_IMM(0xc0010000, REG_W1, imm);
	1465	+ /* ngr %w1,%dst */
	1466	+ EMIT4(0xb9800000, REG_W1, dst_reg);
	1467	+ }
1150	1468	goto branch_oc;
1151	1469
1152	1470	case BPF_JMP \| BPF_JSGT \| BPF_X: /* ((s64) dst > (s64) src) */
	1471	+ case BPF_JMP32 \| BPF_JSGT \| BPF_X: /* ((s32) dst > (s32) src) */
1153	1472	mask = 0x2000; /* jh */
1154	1473	goto branch_xs;
1155	1474	case BPF_JMP \| BPF_JSLT \| BPF_X: /* ((s64) dst < (s64) src) */
	1475	+ case BPF_JMP32 \| BPF_JSLT \| BPF_X: /* ((s32) dst < (s32) src) */
1156	1476	mask = 0x4000; /* jl */
1157	1477	goto branch_xs;
1158	1478	case BPF_JMP \| BPF_JSGE \| BPF_X: /* ((s64) dst >= (s64) src) */
	1479	+ case BPF_JMP32 \| BPF_JSGE \| BPF_X: /* ((s32) dst >= (s32) src) */
1159	1480	mask = 0xa000; /* jhe */
1160	1481	goto branch_xs;
1161	1482	case BPF_JMP \| BPF_JSLE \| BPF_X: /* ((s64) dst <= (s64) src) */
	1483	+ case BPF_JMP32 \| BPF_JSLE \| BPF_X: /* ((s32) dst <= (s32) src) */
1162	1484	mask = 0xc000; /* jle */
1163	1485	goto branch_xs;
1164	1486	case BPF_JMP \| BPF_JGT \| BPF_X: /* (dst > src) */
	1487	+ case BPF_JMP32 \| BPF_JGT \| BPF_X: /* ((u32) dst > (u32) src) */
1165	1488	mask = 0x2000; /* jh */
1166	1489	goto branch_xu;
1167	1490	case BPF_JMP \| BPF_JLT \| BPF_X: /* (dst < src) */
	1491	+ case BPF_JMP32 \| BPF_JLT \| BPF_X: /* ((u32) dst < (u32) src) */
1168	1492	mask = 0x4000; /* jl */
1169	1493	goto branch_xu;
1170	1494	case BPF_JMP \| BPF_JGE \| BPF_X: /* (dst >= src) */
	1495	+ case BPF_JMP32 \| BPF_JGE \| BPF_X: /* ((u32) dst >= (u32) src) */
1171	1496	mask = 0xa000; /* jhe */
1172	1497	goto branch_xu;
1173	1498	case BPF_JMP \| BPF_JLE \| BPF_X: /* (dst <= src) */
	1499	+ case BPF_JMP32 \| BPF_JLE \| BPF_X: /* ((u32) dst <= (u32) src) */
1174	1500	mask = 0xc000; /* jle */
1175	1501	goto branch_xu;
1176	1502	case BPF_JMP \| BPF_JNE \| BPF_X: /* (dst != src) */
	1503	+ case BPF_JMP32 \| BPF_JNE \| BPF_X: /* ((u32) dst != (u32) src) */
1177	1504	mask = 0x7000; /* jne */
1178	1505	goto branch_xu;
1179	1506	case BPF_JMP \| BPF_JEQ \| BPF_X: /* (dst == src) */
	1507	+ case BPF_JMP32 \| BPF_JEQ \| BPF_X: /* ((u32) dst == (u32) src) */
1180	1508	mask = 0x8000; /* je */
1181	1509	goto branch_xu;
1182	1510	case BPF_JMP \| BPF_JSET \| BPF_X: /* (dst & src) */
	1511	+ case BPF_JMP32 \| BPF_JSET \| BPF_X: /* ((u32) dst & (u32) src) */
	1512	+ {
	1513	+ bool is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
	1514	+
1183	1515	mask = 0x7000; /* jnz */
1184		- /* ngrk %w1,%dst,%src */
1185		- EMIT4_RRF(0xb9e40000, REG_W1, dst_reg, src_reg);
	1516	+ /* nrk or ngrk %w1,%dst,%src */
	1517	+ EMIT4_RRF((is_jmp32 ? 0xb9f40000 : 0xb9e40000),
	1518	+ REG_W1, dst_reg, src_reg);
1186	1519	goto branch_oc;
1187	1520	branch_ks:
1188		- /* lgfi %w1,imm (load sign extend imm) */
1189		- EMIT6_IMM(0xc0010000, REG_W1, imm);
1190		- /* cgrj %dst,%w1,mask,off */
1191		- EMIT6_PCREL(0xec000000, 0x0064, dst_reg, REG_W1, i, off, mask);
	1521	+ is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
	1522	+ /* cfi or cgfi %dst,imm */
	1523	+ EMIT6_IMM(is_jmp32 ? 0xc20d0000 : 0xc20c0000,
	1524	+ dst_reg, imm);
	1525	+ if (!is_first_pass(jit) &&
	1526	+ can_use_rel(jit, addrs[i + off + 1])) {
	1527	+ /* brc mask,off */
	1528	+ EMIT4_PCREL_RIC(0xa7040000,
	1529	+ mask >> 12, addrs[i + off + 1]);
	1530	+ } else {
	1531	+ /* brcl mask,off */
	1532	+ EMIT6_PCREL_RILC(0xc0040000,
	1533	+ mask >> 12, addrs[i + off + 1]);
	1534	+ }
1192	1535	break;
1193	1536	branch_ku:
1194	1537	/* lgfi %w1,imm (load sign extend imm) */
1195		- EMIT6_IMM(0xc0010000, REG_W1, imm);
1196		- /* clgrj %dst,%w1,mask,off */
1197		- EMIT6_PCREL(0xec000000, 0x0065, dst_reg, REG_W1, i, off, mask);
1198		- break;
	1538	+ src_reg = REG_1;
	1539	+ EMIT6_IMM(0xc0010000, src_reg, imm);
	1540	+ goto branch_xu;
1199	1541	branch_xs:
1200		- /* cgrj %dst,%src,mask,off */
1201		- EMIT6_PCREL(0xec000000, 0x0064, dst_reg, src_reg, i, off, mask);
	1542	+ is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
	1543	+ if (!is_first_pass(jit) &&
	1544	+ can_use_rel(jit, addrs[i + off + 1])) {
	1545	+ /* crj or cgrj %dst,%src,mask,off */
	1546	+ EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0076 : 0x0064),
	1547	+ dst_reg, src_reg, i, off, mask);
	1548	+ } else {
	1549	+ /* cr or cgr %dst,%src */
	1550	+ if (is_jmp32)
	1551	+ EMIT2(0x1900, dst_reg, src_reg);
	1552	+ else
	1553	+ EMIT4(0xb9200000, dst_reg, src_reg);
	1554	+ /* brcl mask,off */
	1555	+ EMIT6_PCREL_RILC(0xc0040000,
	1556	+ mask >> 12, addrs[i + off + 1]);
	1557	+ }
1202	1558	break;
1203	1559	branch_xu:
1204		- /* clgrj %dst,%src,mask,off */
1205		- EMIT6_PCREL(0xec000000, 0x0065, dst_reg, src_reg, i, off, mask);
	1560	+ is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
	1561	+ if (!is_first_pass(jit) &&
	1562	+ can_use_rel(jit, addrs[i + off + 1])) {
	1563	+ /* clrj or clgrj %dst,%src,mask,off */
	1564	+ EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0077 : 0x0065),
	1565	+ dst_reg, src_reg, i, off, mask);
	1566	+ } else {
	1567	+ /* clr or clgr %dst,%src */
	1568	+ if (is_jmp32)
	1569	+ EMIT2(0x1500, dst_reg, src_reg);
	1570	+ else
	1571	+ EMIT4(0xb9210000, dst_reg, src_reg);
	1572	+ /* brcl mask,off */
	1573	+ EMIT6_PCREL_RILC(0xc0040000,
	1574	+ mask >> 12, addrs[i + off + 1]);
	1575	+ }
1206	1576	break;
1207	1577	branch_oc:
1208		- /* brc mask,jmp_off (branch instruction needs 4 bytes) */
1209		- jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);
1210		- EMIT4_PCREL(0xa7040000 \| mask << 8, jmp_off);
	1578	+ if (!is_first_pass(jit) &&
	1579	+ can_use_rel(jit, addrs[i + off + 1])) {
	1580	+ /* brc mask,off */
	1581	+ EMIT4_PCREL_RIC(0xa7040000,
	1582	+ mask >> 12, addrs[i + off + 1]);
	1583	+ } else {
	1584	+ /* brcl mask,off */
	1585	+ EMIT6_PCREL_RILC(0xc0040000,
	1586	+ mask >> 12, addrs[i + off + 1]);
	1587	+ }
1211	1588	break;
	1589	+ }
1212	1590	default: /* too complex, give up */
1213	1591	pr_err("Unknown opcode %02x\n", insn->code);
1214	1592	return -1;
1215	1593	}
	1594	+
	1595	+ if (probe_prg != -1) {
	1596	+ /*
	1597	+ * Handlers of certain exceptions leave psw.addr pointing to
	1598	+ * the instruction directly after the failing one. Therefore,
	1599	+ * create two exception table entries and also add a nop in
	1600	+ * case two probing instructions come directly after each
	1601	+ * other.
	1602	+ */
	1603	+ nop_prg = jit->prg;
	1604	+ /* bcr 0,%0 */
	1605	+ _EMIT2(0x0700);
	1606	+ err = bpf_jit_probe_mem(jit, fp, probe_prg, nop_prg);
	1607	+ if (err < 0)
	1608	+ return err;
	1609	+ }
	1610	+
1216	1611	return insn_count;
	1612	+}
	1613	+
	1614	+/*
	1615	+ * Return whether new i-th instruction address does not violate any invariant
	1616	+ */
	1617	+static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i)
	1618	+{
	1619	+ /* On the first pass anything goes */
	1620	+ if (is_first_pass(jit))
	1621	+ return true;
	1622	+
	1623	+ /* The codegen pass must not change anything */
	1624	+ if (is_codegen_pass(jit))
	1625	+ return jit->addrs[i] == jit->prg;
	1626	+
	1627	+ /* Passes in between must not increase code size */
	1628	+ return jit->addrs[i] >= jit->prg;
	1629	+}
	1630	+
	1631	+/*
	1632	+ * Update the address of i-th instruction
	1633	+ */
	1634	+static int bpf_set_addr(struct bpf_jit *jit, int i)
	1635	+{
	1636	+ int delta;
	1637	+
	1638	+ if (is_codegen_pass(jit)) {
	1639	+ delta = jit->prg - jit->addrs[i];
	1640	+ if (delta < 0)
	1641	+ bpf_skip(jit, -delta);
	1642	+ }
	1643	+ if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i)))
	1644	+ return -1;
	1645	+ jit->addrs[i] = jit->prg;
	1646	+ return 0;
1217	1647	}
1218	1648
1219	1649	/*
1220	1650	* Compile eBPF program into s390x code
1221	1651	*/
1222		-static int bpf_jit_prog(struct bpf_jit jit, struct bpf_prog fp)
	1652	+static int bpf_jit_prog(struct bpf_jit jit, struct bpf_prog fp,
	1653	+ bool extra_pass, u32 stack_depth)
1223	1654	{
1224		- int i, insn_count;
	1655	+ int i, insn_count, lit32_size, lit64_size;
1225	1656
1226		- jit->lit = jit->lit_start;
	1657	+ jit->lit32 = jit->lit32_start;
	1658	+ jit->lit64 = jit->lit64_start;
1227	1659	jit->prg = 0;
	1660	+ jit->excnt = 0;
1228	1661
1229		- bpf_jit_prologue(jit, fp->aux->stack_depth);
	1662	+ bpf_jit_prologue(jit, stack_depth);
	1663	+ if (bpf_set_addr(jit, 0) < 0)
	1664	+ return -1;
1230	1665	for (i = 0; i < fp->len; i += insn_count) {
1231		- insn_count = bpf_jit_insn(jit, fp, i);
	1666	+ insn_count = bpf_jit_insn(jit, fp, i, extra_pass, stack_depth);
1232	1667	if (insn_count < 0)
1233	1668	return -1;
1234	1669	/* Next instruction address */
1235		- jit->addrs[i + insn_count] = jit->prg;
	1670	+ if (bpf_set_addr(jit, i + insn_count) < 0)
	1671	+ return -1;
1236	1672	}
1237		- bpf_jit_epilogue(jit, fp->aux->stack_depth);
	1673	+ bpf_jit_epilogue(jit, stack_depth);
1238	1674
1239		- jit->lit_start = jit->prg;
1240		- jit->size = jit->lit;
	1675	+ lit32_size = jit->lit32 - jit->lit32_start;
	1676	+ lit64_size = jit->lit64 - jit->lit64_start;
	1677	+ jit->lit32_start = jit->prg;
	1678	+ if (lit32_size)
	1679	+ jit->lit32_start = ALIGN(jit->lit32_start, 4);
	1680	+ jit->lit64_start = jit->lit32_start + lit32_size;
	1681	+ if (lit64_size)
	1682	+ jit->lit64_start = ALIGN(jit->lit64_start, 8);
	1683	+ jit->size = jit->lit64_start + lit64_size;
1241	1684	jit->size_prg = jit->prg;
	1685	+
	1686	+ if (WARN_ON_ONCE(fp->aux->extable &&
	1687	+ jit->excnt != fp->aux->num_exentries))
	1688	+ /* Verifier bug - too many entries. */
	1689	+ return -1;
	1690	+
1242	1691	return 0;
	1692	+}
	1693	+
	1694	+bool bpf_jit_needs_zext(void)
	1695	+{
	1696	+ return true;
	1697	+}
	1698	+
	1699	+struct s390_jit_data {
	1700	+ struct bpf_binary_header *header;
	1701	+ struct bpf_jit ctx;
	1702	+ int pass;
	1703	+};
	1704	+
	1705	+static struct bpf_binary_header bpf_jit_alloc(struct bpf_jit jit,
	1706	+ struct bpf_prog *fp)
	1707	+{
	1708	+ struct bpf_binary_header *header;
	1709	+ u32 extable_size;
	1710	+ u32 code_size;
	1711	+
	1712	+ /* We need two entries per insn. */
	1713	+ fp->aux->num_exentries *= 2;
	1714	+
	1715	+ code_size = roundup(jit->size,
	1716	+ __alignof__(struct exception_table_entry));
	1717	+ extable_size = fp->aux->num_exentries *
	1718	+ sizeof(struct exception_table_entry);
	1719	+ header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf,
	1720	+ 8, jit_fill_hole);
	1721	+ if (!header)
	1722	+ return NULL;
	1723	+ fp->aux->extable = (struct exception_table_entry *)
	1724	+ (jit->prg_buf + code_size);
	1725	+ return header;
1243	1726	}
1244	1727
1245	1728	/*
..	..	@@ -1247,9 +1730,12 @@
1247	1730	*/
1248	1731	struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp)
1249	1732	{
	1733	+ u32 stack_depth = round_up(fp->aux->stack_depth, 8);
1250	1734	struct bpf_prog tmp, orig_fp = fp;
1251	1735	struct bpf_binary_header *header;
	1736	+ struct s390_jit_data *jit_data;
1252	1737	bool tmp_blinded = false;
	1738	+ bool extra_pass = false;
1253	1739	struct bpf_jit jit;
1254	1740	int pass;
1255	1741
..	..	@@ -1268,11 +1754,28 @@
1268	1754	fp = tmp;
1269	1755	}
1270	1756
	1757	+ jit_data = fp->aux->jit_data;
	1758	+ if (!jit_data) {
	1759	+ jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
	1760	+ if (!jit_data) {
	1761	+ fp = orig_fp;
	1762	+ goto out;
	1763	+ }
	1764	+ fp->aux->jit_data = jit_data;
	1765	+ }
	1766	+ if (jit_data->ctx.addrs) {
	1767	+ jit = jit_data->ctx;
	1768	+ header = jit_data->header;
	1769	+ extra_pass = true;
	1770	+ pass = jit_data->pass + 1;
	1771	+ goto skip_init_ctx;
	1772	+ }
	1773	+
1271	1774	memset(&jit, 0, sizeof(jit));
1272		- jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
	1775	+ jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
1273	1776	if (jit.addrs == NULL) {
1274	1777	fp = orig_fp;
1275		- goto out;
	1778	+ goto free_addrs;
1276	1779	}
1277	1780	/*
1278	1781	* Three initial passes:
..	..	@@ -1280,7 +1783,7 @@
1280	1783	* - 3: Calculate program size and addrs arrray
1281	1784	*/
1282	1785	for (pass = 1; pass <= 3; pass++) {
1283		- if (bpf_jit_prog(&jit, fp)) {
	1786	+ if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {
1284	1787	fp = orig_fp;
1285	1788	goto free_addrs;
1286	1789	}
..	..	@@ -1288,16 +1791,13 @@
1288	1791	/*
1289	1792	* Final pass: Allocate and generate program
1290	1793	*/
1291		- if (jit.size >= BPF_SIZE_MAX) {
1292		- fp = orig_fp;
1293		- goto free_addrs;
1294		- }
1295		- header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole);
	1794	+ header = bpf_jit_alloc(&jit, fp);
1296	1795	if (!header) {
1297	1796	fp = orig_fp;
1298	1797	goto free_addrs;
1299	1798	}
1300		- if (bpf_jit_prog(&jit, fp)) {
	1799	+skip_init_ctx:
	1800	+ if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {
1301	1801	bpf_jit_binary_free(header);
1302	1802	fp = orig_fp;
1303	1803	goto free_addrs;
..	..	@@ -1306,12 +1806,24 @@
1306	1806	bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
1307	1807	print_fn_code(jit.prg_buf, jit.size_prg);
1308	1808	}
1309		- bpf_jit_binary_lock_ro(header);
	1809	+ if (!fp->is_func \|\| extra_pass) {
	1810	+ bpf_jit_binary_lock_ro(header);
	1811	+ } else {
	1812	+ jit_data->header = header;
	1813	+ jit_data->ctx = jit;
	1814	+ jit_data->pass = pass;
	1815	+ }
1310	1816	fp->bpf_func = (void *) jit.prg_buf;
1311	1817	fp->jited = 1;
1312	1818	fp->jited_len = jit.size;
	1819	+
	1820	+ if (!fp->is_func \|\| extra_pass) {
	1821	+ bpf_prog_fill_jited_linfo(fp, jit.addrs + 1);
1313	1822	free_addrs:
1314		- kfree(jit.addrs);
	1823	+ kvfree(jit.addrs);
	1824	+ kfree(jit_data);
	1825	+ fp->aux->jit_data = NULL;
	1826	+ }
1315	1827	out:
1316	1828	if (tmp_blinded)
1317	1829	bpf_jit_prog_release_other(fp, fp == orig_fp ?