~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,24 +1,16 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	Copyright (C) 2002 Richard Henderson
3	4	Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
4	5
5		- This program is free software; you can redistribute it and/or modify
6		- it under the terms of the GNU General Public License as published by
7		- the Free Software Foundation; either version 2 of the License, or
8		- (at your option) any later version.
9		-
10		- This program is distributed in the hope that it will be useful,
11		- but WITHOUT ANY WARRANTY; without even the implied warranty of
12		- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13		- GNU General Public License for more details.
14		-
15		- You should have received a copy of the GNU General Public License
16		- along with this program; if not, write to the Free Software
17		- Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18	6	*/
	7	+
	8	+#define INCLUDE_VERMAGIC
	9	+
19	10	#include <linux/export.h>
20	11	#include <linux/extable.h>
21	12	#include <linux/moduleloader.h>
	13	+#include <linux/module_signature.h>
22	14	#include <linux/trace_events.h>
23	15	#include <linux/init.h>
24	16	#include <linux/kallsyms.h>
..	..	@@ -26,6 +18,7 @@
26	18	#include <linux/fs.h>
27	19	#include <linux/sysfs.h>
28	20	#include <linux/kernel.h>
	21	+#include <linux/kernel_read_file.h>
29	22	#include <linux/slab.h>
30	23	#include <linux/vmalloc.h>
31	24	#include <linux/elf.h>
..	..	@@ -70,15 +63,24 @@
70	63	#define CREATE_TRACE_POINTS
71	64	#include <trace/events/module.h>
72	65
	66	+#undef CREATE_TRACE_POINTS
	67	+#include <trace/hooks/module.h>
	68	+#include <trace/hooks/memory.h>
	69	+
73	70	#ifndef ARCH_SHF_SMALL
74	71	#define ARCH_SHF_SMALL 0
75	72	#endif
76	73
77	74	/*
78	75	* Modules' sections will be aligned on page boundaries
79		- * to ensure complete separation of code and data
	76	+ * to ensure complete separation of code and data, but
	77	+ * only when CONFIG_ARCH_HAS_STRICT_MODULE_RWX=y
80	78	*/
	79	+#ifdef CONFIG_ARCH_HAS_STRICT_MODULE_RWX
81	80	# define debug_align(X) ALIGN(X, PAGE_SIZE)
	81	+#else
	82	+# define debug_align(X) (X)
	83	+#endif
82	84
83	85	/* If this is set, the section belongs in the init part of the module */
84	86	#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1))
..	..	@@ -90,8 +92,12 @@
90	92	* 3) module_addr_min/module_addr_max.
91	93	* (delete and add uses RCU list operations). */
92	94	DEFINE_MUTEX(module_mutex);
93		-EXPORT_SYMBOL_GPL(module_mutex);
94	95	static LIST_HEAD(modules);
	96	+
	97	+/* Work queue for freeing init sections in success case */
	98	+static void do_free_init(struct work_struct *w);
	99	+static DECLARE_WORK(init_free_wq, do_free_init);
	100	+static LLIST_HEAD(init_free_list);
95	101
96	102	#ifdef CONFIG_MODULES_TREE_LOOKUP
97	103
..	..	@@ -216,7 +222,8 @@
216	222	{
217	223	struct module *mod;
218	224
219		- list_for_each_entry_rcu(mod, &modules, list) {
	225	+ list_for_each_entry_rcu(mod, &modules, list,
	226	+ lockdep_is_held(&module_mutex)) {
220	227	if (within_module(addr, mod))
221	228	return mod;
222	229	}
..	..	@@ -454,7 +461,8 @@
454	461	if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data))
455	462	return true;
456	463
457		- list_for_each_entry_rcu(mod, &modules, list) {
	464	+ list_for_each_entry_rcu(mod, &modules, list,
	465	+ lockdep_is_held(&module_mutex)) {
458	466	struct symsearch arr[] = {
459	467	{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
460	468	NOT_GPL_ONLY, false },
..	..	@@ -499,9 +507,9 @@
499	507	enum mod_license license;
500	508	};
501	509
502		-static bool check_symbol(const struct symsearch *syms,
503		- struct module *owner,
504		- unsigned int symnum, void *data)
	510	+static bool check_exported_symbol(const struct symsearch *syms,
	511	+ struct module *owner,
	512	+ unsigned int symnum, void *data)
505	513	{
506	514	struct find_symbol_arg *fsa = data;
507	515
..	..	@@ -552,17 +560,25 @@
552	560	#endif
553	561	}
554	562
555		-static int cmp_name(const void va, const void vb)
	563	+static const char kernel_symbol_namespace(const struct kernel_symbol sym)
556	564	{
557		- const char *a;
558		- const struct kernel_symbol *b;
559		- a = va; b = vb;
560		- return strcmp(a, kernel_symbol_name(b));
	565	+#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
	566	+ if (!sym->namespace_offset)
	567	+ return NULL;
	568	+ return offset_to_ptr(&sym->namespace_offset);
	569	+#else
	570	+ return sym->namespace;
	571	+#endif
561	572	}
562	573
563		-static bool find_symbol_in_section(const struct symsearch *syms,
564		- struct module *owner,
565		- void *data)
	574	+static int cmp_name(const void name, const void sym)
	575	+{
	576	+ return strcmp(name, kernel_symbol_name(sym));
	577	+}
	578	+
	579	+static bool find_exported_symbol_in_section(const struct symsearch *syms,
	580	+ struct module *owner,
	581	+ void *data)
566	582	{
567	583	struct find_symbol_arg *fsa = data;
568	584	struct kernel_symbol *sym;
..	..	@@ -570,13 +586,14 @@
570	586	sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
571	587	sizeof(struct kernel_symbol), cmp_name);
572	588
573		- if (sym != NULL && check_symbol(syms, owner, sym - syms->start, data))
	589	+ if (sym != NULL && check_exported_symbol(syms, owner,
	590	+ sym - syms->start, data))
574	591	return true;
575	592
576	593	return false;
577	594	}
578	595
579		-/* Find a symbol and return it, along with, (optional) crc and
	596	+/* Find an exported symbol and return it, along with, (optional) crc and
580	597	* (optional) module which owns it. Needs preempt disabled or module_mutex. */
581	598	static const struct kernel_symbol find_symbol(const char name,
582	599	struct module **owner,
..	..	@@ -591,7 +608,7 @@
591	608	fsa.gplok = gplok;
592	609	fsa.warn = warn;
593	610
594		- if (each_symbol_section(find_symbol_in_section, &fsa)) {
	611	+ if (each_symbol_section(find_exported_symbol_in_section, &fsa)) {
595	612	if (owner)
596	613	*owner = fsa.owner;
597	614	if (crc)
..	..	@@ -616,7 +633,8 @@
616	633
617	634	module_assert_mutex_or_preempt();
618	635
619		- list_for_each_entry_rcu(mod, &modules, list) {
	636	+ list_for_each_entry_rcu(mod, &modules, list,
	637	+ lockdep_is_held(&module_mutex)) {
620	638	if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
621	639	continue;
622	640	if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
..	..	@@ -630,7 +648,6 @@
630	648	module_assert_mutex();
631	649	return find_module_all(name, strlen(name), false);
632	650	}
633		-EXPORT_SYMBOL_GPL(find_module);
634	651
635	652	#ifdef CONFIG_SMP
636	653
..	..	@@ -796,6 +813,7 @@
796	813
797	814	MODINFO_ATTR(version);
798	815	MODINFO_ATTR(srcversion);
	816	+MODINFO_ATTR(scmversion);
799	817
800	818	static char last_unloaded_module[MODULE_NAME_LEN+1];
801	819
..	..	@@ -1258,6 +1276,7 @@
1258	1276	&module_uevent,
1259	1277	&modinfo_version,
1260	1278	&modinfo_srcversion,
	1279	+ &modinfo_scmversion,
1261	1280	&modinfo_initstate,
1262	1281	&modinfo_coresize,
1263	1282	&modinfo_initsize,
..	..	@@ -1388,6 +1407,59 @@
1388	1407	}
1389	1408	#endif /* CONFIG_MODVERSIONS */
1390	1409
	1410	+static char get_modinfo(const struct load_info info, const char *tag);
	1411	+static char get_next_modinfo(const struct load_info info, const char *tag,
	1412	+ char *prev);
	1413	+
	1414	+static int verify_namespace_is_imported(const struct load_info *info,
	1415	+ const struct kernel_symbol *sym,
	1416	+ struct module *mod)
	1417	+{
	1418	+ const char *namespace;
	1419	+ char *imported_namespace;
	1420	+
	1421	+ namespace = kernel_symbol_namespace(sym);
	1422	+ if (namespace && namespace[0]) {
	1423	+ imported_namespace = get_modinfo(info, "import_ns");
	1424	+ while (imported_namespace) {
	1425	+ if (strcmp(namespace, imported_namespace) == 0)
	1426	+ return 0;
	1427	+ imported_namespace = get_next_modinfo(
	1428	+ info, "import_ns", imported_namespace);
	1429	+ }
	1430	+#ifdef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
	1431	+ pr_warn(
	1432	+#else
	1433	+ pr_err(
	1434	+#endif
	1435	+ "%s: module uses symbol (%s) from namespace %s, but does not import it.\n",
	1436	+ mod->name, kernel_symbol_name(sym), namespace);
	1437	+#ifndef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
	1438	+ return -EINVAL;
	1439	+#endif
	1440	+ }
	1441	+ return 0;
	1442	+}
	1443	+
	1444	+static bool inherit_taint(struct module mod, struct module owner)
	1445	+{
	1446	+ if (!owner \|\| !test_bit(TAINT_PROPRIETARY_MODULE, &owner->taints))
	1447	+ return true;
	1448	+
	1449	+ if (mod->using_gplonly_symbols) {
	1450	+ pr_err("%s: module using GPL-only symbols uses symbols from proprietary module %s.\n",
	1451	+ mod->name, owner->name);
	1452	+ return false;
	1453	+ }
	1454	+
	1455	+ if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) {
	1456	+ pr_warn("%s: module uses symbols from proprietary module %s, inheriting taint.\n",
	1457	+ mod->name, owner->name);
	1458	+ set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints);
	1459	+ }
	1460	+ return true;
	1461	+}
	1462	+
1391	1463	/* Resolve a symbol for this module. I.e. if we find one, record usage. */
1392	1464	static const struct kernel_symbol resolve_symbol(struct module mod,
1393	1465	const struct load_info *info,
..	..	@@ -1412,8 +1484,22 @@
1412	1484	if (!sym)
1413	1485	goto unlock;
1414	1486
	1487	+ if (license == GPL_ONLY)
	1488	+ mod->using_gplonly_symbols = true;
	1489	+
	1490	+ if (!inherit_taint(mod, owner)) {
	1491	+ sym = NULL;
	1492	+ goto getname;
	1493	+ }
	1494	+
1415	1495	if (!check_version(info, name, mod, crc)) {
1416	1496	sym = ERR_PTR(-EINVAL);
	1497	+ goto getname;
	1498	+ }
	1499	+
	1500	+ err = verify_namespace_is_imported(info, sym, mod);
	1501	+ if (err) {
	1502	+ sym = ERR_PTR(err);
1417	1503	goto getname;
1418	1504	}
1419	1505
..	..	@@ -1469,7 +1555,7 @@
1469	1555	struct module_sect_attrs {
1470	1556	struct attribute_group grp;
1471	1557	unsigned int nsections;
1472		- struct module_sect_attr attrs[0];
	1558	+ struct module_sect_attr attrs[];
1473	1559	};
1474	1560
1475	1561	#define MODULE_SECT_READ_SIZE (3 /* "0x", "\n" */ + (BITS_PER_LONG / 4))
..	..	@@ -1582,7 +1668,7 @@
1582	1668	struct module_notes_attrs {
1583	1669	struct kobject *dir;
1584	1670	unsigned int notes;
1585		- struct bin_attribute attrs[0];
	1671	+ struct bin_attribute attrs[];
1586	1672	};
1587	1673
1588	1674	static ssize_t module_notes_read(struct file filp, struct kobject kobj,
..	..	@@ -1915,7 +2001,6 @@
1915	2001	mod_sysfs_fini(mod);
1916	2002	}
1917	2003
1918		-#ifdef CONFIG_ARCH_HAS_STRICT_MODULE_RWX
1919	2004	/*
1920	2005	* LKM RO/NX protection: protect module's text/ro-data
1921	2006	* from modification and any data from execution.
..	..	@@ -1929,6 +2014,14 @@
1929	2014	*
1930	2015	* These values are always page-aligned (as is base)
1931	2016	*/
	2017	+
	2018	+/*
	2019	+ * Since some arches are moving towards PAGE_KERNEL module allocations instead
	2020	+ * of PAGE_KERNEL_EXEC, keep frob_text() and module_enable_x() outside of the
	2021	+ * CONFIG_STRICT_MODULE_RWX block below because they are needed regardless of
	2022	+ * whether we are strict.
	2023	+ */
	2024	+#ifdef CONFIG_ARCH_HAS_STRICT_MODULE_RWX
1932	2025	static void frob_text(const struct module_layout *layout,
1933	2026	int (*set_memory)(unsigned long start, int num_pages))
1934	2027	{
..	..	@@ -1937,6 +2030,15 @@
1937	2030	set_memory((unsigned long)layout->base,
1938	2031	layout->text_size >> PAGE_SHIFT);
1939	2032	}
	2033	+
	2034	+static void module_enable_x(const struct module *mod)
	2035	+{
	2036	+ frob_text(&mod->core_layout, set_memory_x);
	2037	+ frob_text(&mod->init_layout, set_memory_x);
	2038	+}
	2039	+#else /* !CONFIG_ARCH_HAS_STRICT_MODULE_RWX */
	2040	+static void module_enable_x(const struct module *mod) { }
	2041	+#endif /* CONFIG_ARCH_HAS_STRICT_MODULE_RWX */
1940	2042
1941	2043	#ifdef CONFIG_STRICT_MODULE_RWX
1942	2044	static void frob_rodata(const struct module_layout *layout,
..	..	@@ -1969,24 +2071,13 @@
1969	2071	(layout->size - layout->ro_after_init_size) >> PAGE_SHIFT);
1970	2072	}
1971	2073
1972		-/* livepatching wants to disable read-only so it can frob module. */
1973		-void module_disable_ro(const struct module *mod)
	2074	+static void module_enable_ro(const struct module *mod, bool after_init)
1974	2075	{
1975	2076	if (!rodata_enabled)
1976	2077	return;
1977	2078
1978		- frob_text(&mod->core_layout, set_memory_rw);
1979		- frob_rodata(&mod->core_layout, set_memory_rw);
1980		- frob_ro_after_init(&mod->core_layout, set_memory_rw);
1981		- frob_text(&mod->init_layout, set_memory_rw);
1982		- frob_rodata(&mod->init_layout, set_memory_rw);
1983		-}
1984		-
1985		-void module_enable_ro(const struct module *mod, bool after_init)
1986		-{
1987		- if (!rodata_enabled)
1988		- return;
1989		-
	2079	+ set_vm_flush_reset_perms(mod->core_layout.base);
	2080	+ set_vm_flush_reset_perms(mod->init_layout.base);
1990	2081	frob_text(&mod->core_layout, set_memory_ro);
1991	2082
1992	2083	frob_rodata(&mod->core_layout, set_memory_ro);
..	..	@@ -2006,88 +2097,32 @@
2006	2097	frob_writable_data(&mod->init_layout, set_memory_nx);
2007	2098	}
2008	2099
2009		-static void module_disable_nx(const struct module *mod)
	2100	+static int module_enforce_rwx_sections(Elf_Ehdr hdr, Elf_Shdr sechdrs,
	2101	+ char secstrings, struct module mod)
2010	2102	{
2011		- frob_rodata(&mod->core_layout, set_memory_x);
2012		- frob_ro_after_init(&mod->core_layout, set_memory_x);
2013		- frob_writable_data(&mod->core_layout, set_memory_x);
2014		- frob_rodata(&mod->init_layout, set_memory_x);
2015		- frob_writable_data(&mod->init_layout, set_memory_x);
2016		-}
	2103	+ const unsigned long shf_wx = SHF_WRITE\|SHF_EXECINSTR;
	2104	+ int i;
2017	2105
2018		-/* Iterate through all modules and set each module's text as RW */
2019		-void set_all_modules_text_rw(void)
2020		-{
2021		- struct module *mod;
2022		-
2023		- if (!rodata_enabled)
2024		- return;
2025		-
2026		- mutex_lock(&module_mutex);
2027		- list_for_each_entry_rcu(mod, &modules, list) {
2028		- if (mod->state == MODULE_STATE_UNFORMED)
2029		- continue;
2030		-
2031		- frob_text(&mod->core_layout, set_memory_rw);
2032		- frob_text(&mod->init_layout, set_memory_rw);
	2106	+ for (i = 0; i < hdr->e_shnum; i++) {
	2107	+ if ((sechdrs[i].sh_flags & shf_wx) == shf_wx) {
	2108	+ pr_err("%s: section %s (index %d) has invalid WRITE\|EXEC flags\n",
	2109	+ mod->name, secstrings + sechdrs[i].sh_name, i);
	2110	+ return -ENOEXEC;
	2111	+ }
2033	2112	}
2034		- mutex_unlock(&module_mutex);
2035		-}
2036	2113
2037		-/* Iterate through all modules and set each module's text as RO */
2038		-void set_all_modules_text_ro(void)
2039		-{
2040		- struct module *mod;
2041		-
2042		- if (!rodata_enabled)
2043		- return;
2044		-
2045		- mutex_lock(&module_mutex);
2046		- list_for_each_entry_rcu(mod, &modules, list) {
2047		- /*
2048		- * Ignore going modules since it's possible that ro
2049		- * protection has already been disabled, otherwise we'll
2050		- * run into protection faults at module deallocation.
2051		- */
2052		- if (mod->state == MODULE_STATE_UNFORMED \|\|
2053		- mod->state == MODULE_STATE_GOING)
2054		- continue;
2055		-
2056		- frob_text(&mod->core_layout, set_memory_ro);
2057		- frob_text(&mod->init_layout, set_memory_ro);
2058		- }
2059		- mutex_unlock(&module_mutex);
2060		-}
2061		-
2062		-static void disable_ro_nx(const struct module_layout *layout)
2063		-{
2064		- if (rodata_enabled) {
2065		- frob_text(layout, set_memory_rw);
2066		- frob_rodata(layout, set_memory_rw);
2067		- frob_ro_after_init(layout, set_memory_rw);
2068		- }
2069		- frob_rodata(layout, set_memory_x);
2070		- frob_ro_after_init(layout, set_memory_x);
2071		- frob_writable_data(layout, set_memory_x);
	2114	+ return 0;
2072	2115	}
2073	2116
2074	2117	#else /* !CONFIG_STRICT_MODULE_RWX */
2075		-static void disable_ro_nx(const struct module_layout *layout) { }
2076	2118	static void module_enable_nx(const struct module *mod) { }
2077		-static void module_disable_nx(const struct module *mod) { }
2078		-#endif /* CONFIG_STRICT_MODULE_RWX */
2079		-
2080		-static void module_enable_x(const struct module *mod)
	2119	+static void module_enable_ro(const struct module *mod, bool after_init) {}
	2120	+static int module_enforce_rwx_sections(Elf_Ehdr hdr, Elf_Shdr sechdrs,
	2121	+ char secstrings, struct module mod)
2081	2122	{
2082		- frob_text(&mod->core_layout, set_memory_x);
2083		- frob_text(&mod->init_layout, set_memory_x);
	2123	+ return 0;
2084	2124	}
2085		-#else /* !CONFIG_ARCH_HAS_STRICT_MODULE_RWX */
2086		-static void disable_ro_nx(const struct module_layout *layout) { }
2087		-static void module_enable_nx(const struct module *mod) { }
2088		-static void module_disable_nx(const struct module *mod) { }
2089		-static void module_enable_x(const struct module *mod) { }
2090		-#endif /* CONFIG_ARCH_HAS_STRICT_MODULE_RWX */
	2125	+#endif /* CONFIG_STRICT_MODULE_RWX */
2091	2126
2092	2127	#ifdef CONFIG_LIVEPATCH
2093	2128	/*
..	..	@@ -2166,6 +2201,11 @@
2166	2201
2167	2202	void __weak module_memfree(void *module_region)
2168	2203	{
	2204	+ /*
	2205	+ * This memory may be RO, and freeing RO memory in an interrupt is not
	2206	+ * supported by vmalloc.
	2207	+ */
	2208	+ WARN_ON(in_interrupt());
2169	2209	vfree(module_region);
2170	2210	}
2171	2211
..	..	@@ -2215,16 +2255,18 @@
2215	2255	/* Remove this module from bug list, this uses list_del_rcu */
2216	2256	module_bug_cleanup(mod);
2217	2257	/* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
2218		- synchronize_sched();
	2258	+ synchronize_rcu();
2219	2259	mutex_unlock(&module_mutex);
2220		-
2221		- /* This may be empty, but that's OK */
2222		- disable_ro_nx(&mod->init_layout);
2223	2260
2224	2261	/* Clean up CFI for the module. */
2225	2262	cfi_cleanup(mod);
2226	2263
	2264	+ /* This may be empty, but that's OK */
2227	2265	module_arch_freeing_init(mod);
	2266	+ trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base,
	2267	+ (mod->init_layout.size)>>PAGE_SHIFT);
	2268	+ trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base,
	2269	+ (mod->init_layout.size)>>PAGE_SHIFT);
2228	2270	module_memfree(mod->init_layout.base);
2229	2271	kfree(mod->args);
2230	2272	percpu_modfree(mod);
..	..	@@ -2233,7 +2275,10 @@
2233	2275	lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
2234	2276
2235	2277	/* Finally, free the core (containing the module structure) */
2236		- disable_ro_nx(&mod->core_layout);
	2278	+ trace_android_vh_set_memory_rw((unsigned long)mod->core_layout.base,
	2279	+ (mod->core_layout.size)>>PAGE_SHIFT);
	2280	+ trace_android_vh_set_memory_nx((unsigned long)mod->core_layout.base,
	2281	+ (mod->core_layout.size)>>PAGE_SHIFT);
2237	2282	module_memfree(mod->core_layout.base);
2238	2283	}
2239	2284
..	..	@@ -2252,13 +2297,22 @@
2252	2297	}
2253	2298	EXPORT_SYMBOL_GPL(__symbol_get);
2254	2299
	2300	+static bool module_init_layout_section(const char *sname)
	2301	+{
	2302	+#ifndef CONFIG_MODULE_UNLOAD
	2303	+ if (module_exit_section(sname))
	2304	+ return true;
	2305	+#endif
	2306	+ return module_init_section(sname);
	2307	+}
	2308	+
2255	2309	/*
2256	2310	* Ensure that an exported symbol [global namespace] does not already exist
2257	2311	* in the kernel or in some other module's exported symbol table.
2258	2312	*
2259	2313	* You must hold the module_mutex.
2260	2314	*/
2261		-static int verify_export_symbols(struct module *mod)
	2315	+static int verify_exported_symbols(struct module *mod)
2262	2316	{
2263	2317	unsigned int i;
2264	2318	struct module *owner;
..	..	@@ -2393,11 +2447,13 @@
2393	2447	if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
2394	2448	continue;
2395	2449
2396		- /* Livepatch relocation sections are applied by livepatch */
2397	2450	if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH)
2398		- continue;
2399		-
2400		- if (info->sechdrs[i].sh_type == SHT_REL)
	2451	+ err = klp_apply_section_relocs(mod, info->sechdrs,
	2452	+ info->secstrings,
	2453	+ info->strtab,
	2454	+ info->index.sym, i,
	2455	+ NULL);
	2456	+ else if (info->sechdrs[i].sh_type == SHT_REL)
2401	2457	err = apply_relocate(info->sechdrs, info->strtab,
2402	2458	info->index.sym, i, mod);
2403	2459	else if (info->sechdrs[i].sh_type == SHT_RELA)
..	..	@@ -2459,7 +2515,7 @@
2459	2515	if ((s->sh_flags & masks[m][0]) != masks[m][0]
2460	2516	\|\| (s->sh_flags & masks[m][1])
2461	2517	\|\| s->sh_entsize != ~0UL
2462		- \|\| strstarts(sname, ".init"))
	2518	+ \|\| module_init_layout_section(sname))
2463	2519	continue;
2464	2520	s->sh_entsize = get_offset(mod, &mod->core_layout.size, s, i);
2465	2521	pr_debug("\t%s\n", sname);
..	..	@@ -2492,7 +2548,7 @@
2492	2548	if ((s->sh_flags & masks[m][0]) != masks[m][0]
2493	2549	\|\| (s->sh_flags & masks[m][1])
2494	2550	\|\| s->sh_entsize != ~0UL
2495		- \|\| !strstarts(sname, ".init"))
	2551	+ \|\| !module_init_layout_section(sname))
2496	2552	continue;
2497	2553	s->sh_entsize = (get_offset(mod, &mod->init_layout.size, s, i)
2498	2554	\| INIT_OFFSET_MASK);
..	..	@@ -2554,7 +2610,8 @@
2554	2610	return string;
2555	2611	}
2556	2612
2557		-static char get_modinfo(struct load_info info, const char *tag)
	2613	+static char get_next_modinfo(const struct load_info info, const char *tag,
	2614	+ char *prev)
2558	2615	{
2559	2616	char *p;
2560	2617	unsigned int taglen = strlen(tag);
..	..	@@ -2565,11 +2622,23 @@
2565	2622	* get_modinfo() calls made before rewrite_section_headers()
2566	2623	* must use sh_offset, as sh_addr isn't set!
2567	2624	*/
2568		- for (p = (char *)info->hdr + infosec->sh_offset; p; p = next_string(p, &size)) {
	2625	+ char modinfo = (char )info->hdr + infosec->sh_offset;
	2626	+
	2627	+ if (prev) {
	2628	+ size -= prev - modinfo;
	2629	+ modinfo = next_string(prev, &size);
	2630	+ }
	2631	+
	2632	+ for (p = modinfo; p; p = next_string(p, &size)) {
2569	2633	if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
2570	2634	return p + taglen + 1;
2571	2635	}
2572	2636	return NULL;
	2637	+}
	2638	+
	2639	+static char get_modinfo(const struct load_info info, const char *tag)
	2640	+{
	2641	+ return get_next_modinfo(info, tag, NULL);
2573	2642	}
2574	2643
2575	2644	static void setup_modinfo(struct module mod, struct load_info info)
..	..	@@ -2596,10 +2665,10 @@
2596	2665
2597	2666	#ifdef CONFIG_KALLSYMS
2598	2667
2599		-/* lookup symbol in given range of kernel_symbols */
2600		-static const struct kernel_symbol lookup_symbol(const char name,
2601		- const struct kernel_symbol *start,
2602		- const struct kernel_symbol *stop)
	2668	+/* Lookup exported symbol in given range of kernel_symbols */
	2669	+static const struct kernel_symbol lookup_exported_symbol(const char name,
	2670	+ const struct kernel_symbol *start,
	2671	+ const struct kernel_symbol *stop)
2603	2672	{
2604	2673	return bsearch(name, start, stop - start,
2605	2674	sizeof(struct kernel_symbol), cmp_name);
..	..	@@ -2610,9 +2679,10 @@
2610	2679	{
2611	2680	const struct kernel_symbol *ks;
2612	2681	if (!mod)
2613		- ks = lookup_symbol(name, __start___ksymtab, __stop___ksymtab);
	2682	+ ks = lookup_exported_symbol(name, __start___ksymtab, __stop___ksymtab);
2614	2683	else
2615		- ks = lookup_symbol(name, mod->syms, mod->syms + mod->num_syms);
	2684	+ ks = lookup_exported_symbol(name, mod->syms, mod->syms + mod->num_syms);
	2685	+
2616	2686	return ks != NULL && kernel_symbol_value(ks) == value;
2617	2687	}
2618	2688
..	..	@@ -2720,6 +2790,8 @@
2720	2790	info->symoffs = ALIGN(mod->core_layout.size, symsect->sh_addralign ?: 1);
2721	2791	info->stroffs = mod->core_layout.size = info->symoffs + ndst * sizeof(Elf_Sym);
2722	2792	mod->core_layout.size += strtab_size;
	2793	+ info->core_typeoffs = mod->core_layout.size;
	2794	+ mod->core_layout.size += ndst * sizeof(char);
2723	2795	mod->core_layout.size = debug_align(mod->core_layout.size);
2724	2796
2725	2797	/* Put string table section at end of init part of module. */
..	..	@@ -2733,6 +2805,8 @@
2733	2805	__alignof__(struct mod_kallsyms));
2734	2806	info->mod_kallsyms_init_off = mod->init_layout.size;
2735	2807	mod->init_layout.size += sizeof(struct mod_kallsyms);
	2808	+ info->init_typeoffs = mod->init_layout.size;
	2809	+ mod->init_layout.size += nsrc * sizeof(char);
2736	2810	mod->init_layout.size = debug_align(mod->init_layout.size);
2737	2811	}
2738	2812
..	..	@@ -2756,20 +2830,23 @@
2756	2830	mod->kallsyms->num_symtab = symsec->sh_size / sizeof(Elf_Sym);
2757	2831	/* Make sure we get permanent strtab: don't use info->strtab. */
2758	2832	mod->kallsyms->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
	2833	+ mod->kallsyms->typetab = mod->init_layout.base + info->init_typeoffs;
2759	2834
2760		- /* Set types up while we still have access to sections. */
2761		- for (i = 0; i < mod->kallsyms->num_symtab; i++)
2762		- mod->kallsyms->symtab[i].st_info
2763		- = elf_type(&mod->kallsyms->symtab[i], info);
2764		-
2765		- /* Now populate the cut down core kallsyms for after init. */
	2835	+ /*
	2836	+ * Now populate the cut down core kallsyms for after init
	2837	+ * and set types up while we still have access to sections.
	2838	+ */
2766	2839	mod->core_kallsyms.symtab = dst = mod->core_layout.base + info->symoffs;
2767	2840	mod->core_kallsyms.strtab = s = mod->core_layout.base + info->stroffs;
	2841	+ mod->core_kallsyms.typetab = mod->core_layout.base + info->core_typeoffs;
2768	2842	src = mod->kallsyms->symtab;
2769	2843	for (ndst = i = 0; i < mod->kallsyms->num_symtab; i++) {
	2844	+ mod->kallsyms->typetab[i] = elf_type(src + i, info);
2770	2845	if (i == 0 \|\| is_livepatch_module(mod) \|\|
2771	2846	is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum,
2772	2847	info->index.pcpu)) {
	2848	+ mod->core_kallsyms.typetab[ndst] =
	2849	+ mod->kallsyms->typetab[i];
2773	2850	dst[ndst] = src[i];
2774	2851	dst[ndst++].st_name = s - mod->core_kallsyms.strtab;
2775	2852	s += strlcpy(s, &mod->kallsyms->strtab[src[i].st_name],
..	..	@@ -2792,11 +2869,7 @@
2792	2869	{
2793	2870	if (!debug)
2794	2871	return;
2795		-#ifdef CONFIG_DYNAMIC_DEBUG
2796		- if (ddebug_add_module(debug, num, mod->name))
2797		- pr_err("dynamic debug error adding module: %s\n",
2798		- debug->modname);
2799		-#endif
	2872	+ ddebug_add_module(debug, num, mod->name);
2800	2873	}
2801	2874
2802	2875	static void dynamic_debug_remove(struct module mod, struct _ddebug debug)
..	..	@@ -2807,7 +2880,19 @@
2807	2880
2808	2881	void * __weak module_alloc(unsigned long size)
2809	2882	{
2810		- return vmalloc_exec(size);
	2883	+ return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
	2884	+ GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS,
	2885	+ NUMA_NO_NODE, __builtin_return_address(0));
	2886	+}
	2887	+
	2888	+bool __weak module_init_section(const char *name)
	2889	+{
	2890	+ return strstarts(name, ".init");
	2891	+}
	2892	+
	2893	+bool __weak module_exit_section(const char *name)
	2894	+{
	2895	+ return strstarts(name, ".exit");
2811	2896	}
2812	2897
2813	2898	#ifdef CONFIG_DEBUG_KMEMLEAK
..	..	@@ -2840,8 +2925,9 @@
2840	2925	#ifdef CONFIG_MODULE_SIG
2841	2926	static int module_sig_check(struct load_info *info, int flags)
2842	2927	{
2843		- int err = -ENOKEY;
	2928	+ int err = -ENODATA;
2844	2929	const unsigned long markerlen = sizeof(MODULE_SIG_STRING) - 1;
	2930	+ const char *reason;
2845	2931	const void *mod = info->hdr;
2846	2932
2847	2933	/*
..	..	@@ -2856,16 +2942,39 @@
2856	2942	err = mod_verify_sig(mod, info);
2857	2943	}
2858	2944
2859		- if (!err) {
	2945	+ switch (err) {
	2946	+ case 0:
2860	2947	info->sig_ok = true;
2861	2948	return 0;
	2949	+
	2950	+ /* We don't permit modules to be loaded into trusted kernels
	2951	+ * without a valid signature on them, but if we're not
	2952	+ * enforcing, certain errors are non-fatal.
	2953	+ */
	2954	+ case -ENODATA:
	2955	+ reason = "unsigned module";
	2956	+ break;
	2957	+ case -ENOPKG:
	2958	+ reason = "module with unsupported crypto";
	2959	+ break;
	2960	+ case -ENOKEY:
	2961	+ reason = "module with unavailable key";
	2962	+ break;
	2963	+
	2964	+ /* All other errors are fatal, including nomem, unparseable
	2965	+ * signatures and signature check failures - even if signatures
	2966	+ * aren't required.
	2967	+ */
	2968	+ default:
	2969	+ return err;
2862	2970	}
2863	2971
2864		- /* Not having a signature is only an error if we're strict. */
2865		- if (err == -ENOKEY && !is_module_sig_enforced())
2866		- err = 0;
	2972	+ if (is_module_sig_enforced()) {
	2973	+ pr_notice("Loading of %s is rejected\n", reason);
	2974	+ return -EKEYREJECTED;
	2975	+ }
2867	2976
2868		- return err;
	2977	+ return security_locked_down(LOCKDOWN_MODULE_SIGNATURE);
2869	2978	}
2870	2979	#else /* !CONFIG_MODULE_SIG */
2871	2980	static int module_sig_check(struct load_info *info, int flags)
..	..	@@ -2874,9 +2983,33 @@
2874	2983	}
2875	2984	#endif /* !CONFIG_MODULE_SIG */
2876	2985
2877		-/* Sanity checks against invalid binaries, wrong arch, weird elf version. */
2878		-static int elf_header_check(struct load_info *info)
	2986	+static int validate_section_offset(struct load_info info, Elf_Shdr shdr)
2879	2987	{
	2988	+ unsigned long secend;
	2989	+
	2990	+ /*
	2991	+ * Check for both overflow and offset/size being
	2992	+ * too large.
	2993	+ */
	2994	+ secend = shdr->sh_offset + shdr->sh_size;
	2995	+ if (secend < shdr->sh_offset \|\| secend > info->len)
	2996	+ return -ENOEXEC;
	2997	+
	2998	+ return 0;
	2999	+}
	3000	+
	3001	+/*
	3002	+ * Sanity checks against invalid binaries, wrong arch, weird elf version.
	3003	+ *
	3004	+ * Also do basic validity checks against section offsets and sizes, the
	3005	+ * section name string table, and the indices used for it (sh_name).
	3006	+ */
	3007	+static int elf_validity_check(struct load_info *info)
	3008	+{
	3009	+ unsigned int i;
	3010	+ Elf_Shdr shdr, strhdr;
	3011	+ int err;
	3012	+
2880	3013	if (info->len < sizeof(*(info->hdr)))
2881	3014	return -ENOEXEC;
2882	3015
..	..	@@ -2886,10 +3019,77 @@
2886	3019	\|\| info->hdr->e_shentsize != sizeof(Elf_Shdr))
2887	3020	return -ENOEXEC;
2888	3021
	3022	+ /*
	3023	+ * e_shnum is 16 bits, and sizeof(Elf_Shdr) is
	3024	+ * known and small. So e_shnum * sizeof(Elf_Shdr)
	3025	+ * will not overflow unsigned long on any platform.
	3026	+ */
2889	3027	if (info->hdr->e_shoff >= info->len
2890	3028	\|\| (info->hdr->e_shnum * sizeof(Elf_Shdr) >
2891	3029	info->len - info->hdr->e_shoff))
2892	3030	return -ENOEXEC;
	3031	+
	3032	+ info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
	3033	+
	3034	+ /*
	3035	+ * Verify if the section name table index is valid.
	3036	+ */
	3037	+ if (info->hdr->e_shstrndx == SHN_UNDEF
	3038	+ \|\| info->hdr->e_shstrndx >= info->hdr->e_shnum)
	3039	+ return -ENOEXEC;
	3040	+
	3041	+ strhdr = &info->sechdrs[info->hdr->e_shstrndx];
	3042	+ err = validate_section_offset(info, strhdr);
	3043	+ if (err < 0)
	3044	+ return err;
	3045	+
	3046	+ /*
	3047	+ * The section name table must be NUL-terminated, as required
	3048	+ * by the spec. This makes strcmp and pr_* calls that access
	3049	+ * strings in the section safe.
	3050	+ */
	3051	+ info->secstrings = (void *)info->hdr + strhdr->sh_offset;
	3052	+ if (info->secstrings[strhdr->sh_size - 1] != '\0')
	3053	+ return -ENOEXEC;
	3054	+
	3055	+ /*
	3056	+ * The code assumes that section 0 has a length of zero and
	3057	+ * an addr of zero, so check for it.
	3058	+ */
	3059	+ if (info->sechdrs[0].sh_type != SHT_NULL
	3060	+ \|\| info->sechdrs[0].sh_size != 0
	3061	+ \|\| info->sechdrs[0].sh_addr != 0)
	3062	+ return -ENOEXEC;
	3063	+
	3064	+ for (i = 1; i < info->hdr->e_shnum; i++) {
	3065	+ shdr = &info->sechdrs[i];
	3066	+ switch (shdr->sh_type) {
	3067	+ case SHT_NULL:
	3068	+ case SHT_NOBITS:
	3069	+ continue;
	3070	+ case SHT_SYMTAB:
	3071	+ if (shdr->sh_link == SHN_UNDEF
	3072	+ \|\| shdr->sh_link >= info->hdr->e_shnum)
	3073	+ return -ENOEXEC;
	3074	+ fallthrough;
	3075	+ default:
	3076	+ err = validate_section_offset(info, shdr);
	3077	+ if (err < 0) {
	3078	+ pr_err("Invalid ELF section in module (section %u type %u)\n",
	3079	+ i, shdr->sh_type);
	3080	+ return err;
	3081	+ }
	3082	+
	3083	+ if (shdr->sh_flags & SHF_ALLOC) {
	3084	+ if (shdr->sh_name >= strhdr->sh_size) {
	3085	+ pr_err("Invalid ELF section name in module (section %u type %u)\n",
	3086	+ i, shdr->sh_type);
	3087	+ return -ENOEXEC;
	3088	+ }
	3089	+ }
	3090	+ break;
	3091	+ }
	3092	+ }
2893	3093
2894	3094	return 0;
2895	3095	}
..	..	@@ -2955,22 +3155,27 @@
2955	3155	if (info->len < sizeof(*(info->hdr)))
2956	3156	return -ENOEXEC;
2957	3157
2958		- err = security_kernel_load_data(LOADING_MODULE);
	3158	+ err = security_kernel_load_data(LOADING_MODULE, true);
2959	3159	if (err)
2960	3160	return err;
2961	3161
2962	3162	/* Suck in entire file: we'll want most of it. */
2963		- info->hdr = __vmalloc(info->len,
2964		- GFP_KERNEL \| __GFP_NOWARN, PAGE_KERNEL);
	3163	+ info->hdr = __vmalloc(info->len, GFP_KERNEL \| __GFP_NOWARN);
2965	3164	if (!info->hdr)
2966	3165	return -ENOMEM;
2967	3166
2968	3167	if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) {
2969		- vfree(info->hdr);
2970		- return -EFAULT;
	3168	+ err = -EFAULT;
	3169	+ goto out;
2971	3170	}
2972	3171
2973		- return 0;
	3172	+ err = security_kernel_post_load_data((char *)info->hdr, info->len,
	3173	+ LOADING_MODULE, "init_module");
	3174	+out:
	3175	+ if (err)
	3176	+ vfree(info->hdr);
	3177	+
	3178	+ return err;
2974	3179	}
2975	3180
2976	3181	static void free_copy(struct load_info *info)
..	..	@@ -2987,21 +3192,11 @@
2987	3192
2988	3193	for (i = 1; i < info->hdr->e_shnum; i++) {
2989	3194	Elf_Shdr *shdr = &info->sechdrs[i];
2990		- if (shdr->sh_type != SHT_NOBITS
2991		- && info->len < shdr->sh_offset + shdr->sh_size) {
2992		- pr_err("Module len %lu truncated\n", info->len);
2993		- return -ENOEXEC;
2994		- }
2995	3195
2996	3196	/* Mark all sections sh_addr with their address in the
2997	3197	temporary image. */
2998	3198	shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
2999	3199
3000		-#ifndef CONFIG_MODULE_UNLOAD
3001		- /* Don't load .exit sections */
3002		- if (strstarts(info->secstrings+shdr->sh_name, ".exit"))
3003		- shdr->sh_flags &= ~(unsigned long)SHF_ALLOC;
3004		-#endif
3005	3200	}
3006	3201
3007	3202	/* Track but don't keep modinfo and version sections. */
..	..	@@ -3022,11 +3217,6 @@
3022	3217	static int setup_load_info(struct load_info *info, int flags)
3023	3218	{
3024	3219	unsigned int i;
3025		-
3026		- /* Set up the convenience variables */
3027		- info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
3028		- info->secstrings = (void *)info->hdr
3029		- + info->sechdrs[info->hdr->e_shstrndx].sh_offset;
3030	3220
3031	3221	/* Try to find a name early so we can log errors with a module name */
3032	3222	info->index.info = find_sec(info, ".modinfo");
..	..	@@ -3164,10 +3354,23 @@
3164	3354	}
3165	3355	#endif
3166	3356
	3357	+ mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1,
	3358	+ &mod->noinstr_text_size);
	3359	+
3167	3360	#ifdef CONFIG_TRACEPOINTS
3168	3361	mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
3169	3362	sizeof(*mod->tracepoints_ptrs),
3170	3363	&mod->num_tracepoints);
	3364	+#endif
	3365	+#ifdef CONFIG_TREE_SRCU
	3366	+ mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs",
	3367	+ sizeof(*mod->srcu_struct_ptrs),
	3368	+ &mod->num_srcu_structs);
	3369	+#endif
	3370	+#ifdef CONFIG_BPF_EVENTS
	3371	+ mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map",
	3372	+ sizeof(*mod->bpf_raw_events),
	3373	+ &mod->num_bpf_raw_events);
3171	3374	#endif
3172	3375	#ifdef CONFIG_JUMP_LABEL
3173	3376	mod->jump_entries = section_objs(info, "__jump_table",
..	..	@@ -3189,7 +3392,7 @@
3189	3392	#endif
3190	3393	#ifdef CONFIG_FTRACE_MCOUNT_RECORD
3191	3394	/* sechdrs[0].sh_size is always zero */
3192		- mod->ftrace_callsites = section_objs(info, "__mcount_loc",
	3395	+ mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION,
3193	3396	sizeof(*mod->ftrace_callsites),
3194	3397	&mod->num_ftrace_callsites);
3195	3398	#endif
..	..	@@ -3198,13 +3401,25 @@
3198	3401	sizeof(*mod->ei_funcs),
3199	3402	&mod->num_ei_funcs);
3200	3403	#endif
	3404	+#ifdef CONFIG_KPROBES
	3405	+ mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1,
	3406	+ &mod->kprobes_text_size);
	3407	+ mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist",
	3408	+ sizeof(unsigned long),
	3409	+ &mod->num_kprobe_blacklist);
	3410	+#endif
	3411	+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
	3412	+ mod->static_call_sites = section_objs(info, ".static_call_sites",
	3413	+ sizeof(*mod->static_call_sites),
	3414	+ &mod->num_static_call_sites);
	3415	+#endif
3201	3416	mod->extable = section_objs(info, "__ex_table",
3202	3417	sizeof(*mod->extable), &mod->num_exentries);
3203	3418
3204	3419	if (section_addr(info, "__obsparm"))
3205	3420	pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
3206	3421
3207		- info->debug = section_objs(info, "__verbose",
	3422	+ info->debug = section_objs(info, "__dyndbg",
3208	3423	sizeof(*info->debug), &info->num_debug);
3209	3424
3210	3425	return 0;
..	..	@@ -3316,12 +3531,6 @@
3316	3531
3317	3532	static void flush_module_icache(const struct module *mod)
3318	3533	{
3319		- mm_segment_t old_fs;
3320		-
3321		- /* flush the icache in correct context */
3322		- old_fs = get_fs();
3323		- set_fs(KERNEL_DS);
3324		-
3325	3534	/*
3326	3535	* Flush the instruction cache, since we've played with text.
3327	3536	* Do it before processing of module parameters, so the module
..	..	@@ -3333,8 +3542,6 @@
3333	3542	+ mod->init_layout.size);
3334	3543	flush_icache_range((unsigned long)mod->core_layout.base,
3335	3544	(unsigned long)mod->core_layout.base + mod->core_layout.size);
3336		-
3337		- set_fs(old_fs);
3338	3545	}
3339	3546
3340	3547	int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
..	..	@@ -3382,6 +3589,11 @@
3382	3589	if (err < 0)
3383	3590	return ERR_PTR(err);
3384	3591
	3592	+ err = module_enforce_rwx_sections(info->hdr, info->sechdrs,
	3593	+ info->secstrings, info->mod);
	3594	+ if (err < 0)
	3595	+ return ERR_PTR(err);
	3596	+
3385	3597	/* We will do a special allocation for per-cpu sections later. */
3386	3598	info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
3387	3599
..	..	@@ -3391,6 +3603,15 @@
3391	3603	* Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set.
3392	3604	*/
3393	3605	ndx = find_sec(info, ".data..ro_after_init");
	3606	+ if (ndx)
	3607	+ info->sechdrs[ndx].sh_flags \|= SHF_RO_AFTER_INIT;
	3608	+ /*
	3609	+ * Mark the __jump_table section as ro_after_init as well: these data
	3610	+ * structures are never modified, with the exception of entries that
	3611	+ * refer to code in the __init section, which are annotated as such
	3612	+ * at module load time.
	3613	+ */
	3614	+ ndx = find_sec(info, "__jump_table");
3394	3615	if (ndx)
3395	3616	info->sechdrs[ndx].sh_flags \|= SHF_RO_AFTER_INIT;
3396	3617
..	..	@@ -3416,7 +3637,15 @@
3416	3637	{
3417	3638	percpu_modfree(mod);
3418	3639	module_arch_freeing_init(mod);
	3640	+ trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base,
	3641	+ (mod->init_layout.size)>>PAGE_SHIFT);
	3642	+ trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base,
	3643	+ (mod->init_layout.size)>>PAGE_SHIFT);
3419	3644	module_memfree(mod->init_layout.base);
	3645	+ trace_android_vh_set_memory_rw((unsigned long)mod->core_layout.base,
	3646	+ (mod->core_layout.size)>>PAGE_SHIFT);
	3647	+ trace_android_vh_set_memory_nx((unsigned long)mod->core_layout.base,
	3648	+ (mod->core_layout.size)>>PAGE_SHIFT);
3420	3649	module_memfree(mod->core_layout.base);
3421	3650	}
3422	3651
..	..	@@ -3426,8 +3655,6 @@
3426	3655	{
3427	3656	return 0;
3428	3657	}
3429		-
3430		-static void cfi_init(struct module *mod);
3431	3658
3432	3659	static int post_relocation(struct module mod, const struct load_info info)
3433	3660	{
..	..	@@ -3440,9 +3667,6 @@
3440	3667
3441	3668	/* Setup kallsyms-specific fields. */
3442	3669	add_kallsyms(mod, info);
3443		-
3444		- /* Setup CFI for the module. */
3445		- cfi_init(mod);
3446	3670
3447	3671	/* Arch-specific module finalizing. */
3448	3672	return module_finalize(info->hdr, info->sechdrs, mod);
..	..	@@ -3481,15 +3705,24 @@
3481	3705
3482	3706	/* For freeing module_init on success, in case kallsyms traversing */
3483	3707	struct mod_initfree {
3484		- struct rcu_head rcu;
	3708	+ struct llist_node node;
3485	3709	void *module_init;
3486	3710	};
3487	3711
3488		-static void do_free_init(struct rcu_head *head)
	3712	+static void do_free_init(struct work_struct *w)
3489	3713	{
3490		- struct mod_initfree *m = container_of(head, struct mod_initfree, rcu);
3491		- module_memfree(m->module_init);
3492		- kfree(m);
	3714	+ struct llist_node pos, n, *list;
	3715	+ struct mod_initfree *initfree;
	3716	+
	3717	+ list = llist_del_all(&init_free_list);
	3718	+
	3719	+ synchronize_rcu();
	3720	+
	3721	+ llist_for_each_safe(pos, n, list) {
	3722	+ initfree = container_of(pos, struct mod_initfree, node);
	3723	+ module_memfree(initfree->module_init);
	3724	+ kfree(initfree);
	3725	+ }
3493	3726	}
3494	3727
3495	3728	/*
..	..	@@ -3555,9 +3788,13 @@
3555	3788	rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
3556	3789	#endif
3557	3790	module_enable_ro(mod, true);
	3791	+ trace_android_vh_set_module_permit_after_init(mod);
3558	3792	mod_tree_remove_init(mod);
3559		- disable_ro_nx(&mod->init_layout);
3560	3793	module_arch_freeing_init(mod);
	3794	+ trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base,
	3795	+ (mod->init_layout.size)>>PAGE_SHIFT);
	3796	+ trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base,
	3797	+ (mod->init_layout.size)>>PAGE_SHIFT);
3561	3798	mod->init_layout.base = NULL;
3562	3799	mod->init_layout.size = 0;
3563	3800	mod->init_layout.ro_size = 0;
..	..	@@ -3566,15 +3803,19 @@
3566	3803	/*
3567	3804	* We want to free module_init, but be aware that kallsyms may be
3568	3805	* walking this with preempt disabled. In all the failure paths, we
3569		- * call synchronize_sched(), but we don't want to slow down the success
3570		- * path, so use actual RCU here.
	3806	+ * call synchronize_rcu(), but we don't want to slow down the success
	3807	+ * path. module_memfree() cannot be called in an interrupt, so do the
	3808	+ * work and call synchronize_rcu() in a work queue.
	3809	+ *
3571	3810	* Note that module_alloc() on most architectures creates W+X page
3572	3811	* mappings which won't be cleaned up until do_free_init() runs. Any
3573	3812	* code such as mark_rodata_ro() which depends on those mappings to
3574	3813	* be cleaned up needs to sync with the queued work - ie
3575		- * rcu_barrier_sched()
	3814	+ * rcu_barrier()
3576	3815	*/
3577		- call_rcu_sched(&freeinit->rcu, do_free_init);
	3816	+ if (llist_add(&freeinit->node, &init_free_list))
	3817	+ schedule_work(&init_free_wq);
	3818	+
3578	3819	mutex_unlock(&module_mutex);
3579	3820	wake_up_all(&module_wq);
3580	3821
..	..	@@ -3585,7 +3826,7 @@
3585	3826	fail:
3586	3827	/* Try to protect us from buggy refcounters. */
3587	3828	mod->state = MODULE_STATE_GOING;
3588		- synchronize_sched();
	3829	+ synchronize_rcu();
3589	3830	module_put(mod);
3590	3831	blocking_notifier_call_chain(&module_notify_list,
3591	3832	MODULE_STATE_GOING, mod);
..	..	@@ -3650,7 +3891,7 @@
3650	3891	mutex_lock(&module_mutex);
3651	3892
3652	3893	/* Find duplicate symbols (must be called under lock). */
3653		- err = verify_export_symbols(mod);
	3894	+ err = verify_exported_symbols(mod);
3654	3895	if (err < 0)
3655	3896	goto out;
3656	3897
..	..	@@ -3660,6 +3901,7 @@
3660	3901	module_enable_ro(mod, false);
3661	3902	module_enable_nx(mod);
3662	3903	module_enable_x(mod);
	3904	+ trace_android_vh_set_module_permit_before_init(mod);
3663	3905
3664	3906	/* Mark state as coming so strong_try_module_get() ignores us,
3665	3907	* but kallsyms etc. can see us. */
..	..	@@ -3682,9 +3924,13 @@
3682	3924	if (err)
3683	3925	return err;
3684	3926
3685		- blocking_notifier_call_chain(&module_notify_list,
3686		- MODULE_STATE_COMING, mod);
3687		- return 0;
	3927	+ err = blocking_notifier_call_chain_robust(&module_notify_list,
	3928	+ MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
	3929	+ err = notifier_to_errno(err);
	3930	+ if (err)
	3931	+ klp_module_going(mod);
	3932	+
	3933	+ return err;
3688	3934	}
3689	3935
3690	3936	static int unknown_module_param_cb(char param, char val, const char *modname,
..	..	@@ -3705,6 +3951,8 @@
3705	3951	return 0;
3706	3952	}
3707	3953
	3954	+static void cfi_init(struct module *mod);
	3955	+
3708	3956	/* Allocate and load the module: note that size of section 0 is always
3709	3957	zero, and we rely on this for optional sections. */
3710	3958	static int load_module(struct load_info info, const char __user uargs,
..	..	@@ -3714,22 +3962,49 @@
3714	3962	long err = 0;
3715	3963	char *after_dashes;
3716	3964
3717		- err = elf_header_check(info);
	3965	+ /*
	3966	+ * Do the signature check (if any) first. All that
	3967	+ * the signature check needs is info->len, it does
	3968	+ * not need any of the section info. That can be
	3969	+ * set up later. This will minimize the chances
	3970	+ * of a corrupt module causing problems before
	3971	+ * we even get to the signature check.
	3972	+ *
	3973	+ * The check will also adjust info->len by stripping
	3974	+ * off the sig length at the end of the module, making
	3975	+ * checks against info->len more correct.
	3976	+ */
	3977	+ err = module_sig_check(info, flags);
3718	3978	if (err)
3719	3979	goto free_copy;
3720	3980
	3981	+ /*
	3982	+ * Do basic sanity checks against the ELF header and
	3983	+ * sections.
	3984	+ */
	3985	+ err = elf_validity_check(info);
	3986	+ if (err) {
	3987	+ pr_err("Module has invalid ELF structures\n");
	3988	+ goto free_copy;
	3989	+ }
	3990	+
	3991	+ /*
	3992	+ * Everything checks out, so set up the section info
	3993	+ * in the info structure.
	3994	+ */
3721	3995	err = setup_load_info(info, flags);
3722	3996	if (err)
3723	3997	goto free_copy;
3724	3998
	3999	+ /*
	4000	+ * Now that we know we have the correct module name, check
	4001	+ * if it's blacklisted.
	4002	+ */
3725	4003	if (blacklisted(info->name)) {
3726	4004	err = -EPERM;
	4005	+ pr_err("Module %s is blacklisted\n", info->name);
3727	4006	goto free_copy;
3728	4007	}
3729		-
3730		- err = module_sig_check(info, flags);
3731		- if (err)
3732		- goto free_copy;
3733	4008
3734	4009	err = rewrite_section_headers(info, flags);
3735	4010	if (err)
..	..	@@ -3805,6 +4080,9 @@
3805	4080
3806	4081	flush_module_icache(mod);
3807	4082
	4083	+ /* Setup CFI for the module. */
	4084	+ cfi_init(mod);
	4085	+
3808	4086	/* Now copy in args */
3809	4087	mod->args = strndup_user(uargs, ~0UL >> 1);
3810	4088	if (IS_ERR(mod->args)) {
..	..	@@ -3872,16 +4150,13 @@
3872	4150	module_bug_cleanup(mod);
3873	4151	mutex_unlock(&module_mutex);
3874	4152
3875		- /* we can't deallocate the module until we clear memory protection */
3876		- module_disable_ro(mod);
3877		- module_disable_nx(mod);
3878		-
3879	4153	ddebug_cleanup:
3880	4154	ftrace_release_mod(mod);
3881	4155	dynamic_debug_remove(mod, info->debug);
3882		- synchronize_sched();
	4156	+ synchronize_rcu();
3883	4157	kfree(mod->args);
3884	4158	free_arch_cleanup:
	4159	+ cfi_cleanup(mod);
3885	4160	module_arch_cleanup(mod);
3886	4161	free_modinfo:
3887	4162	free_modinfo(mod);
..	..	@@ -3894,7 +4169,7 @@
3894	4169	mod_tree_remove(mod);
3895	4170	wake_up_all(&module_wq);
3896	4171	/* Wait for RCU-sched synchronizing before releasing mod->list. */
3897		- synchronize_sched();
	4172	+ synchronize_rcu();
3898	4173	mutex_unlock(&module_mutex);
3899	4174	free_module:
3900	4175	/* Free lock-classes; relies on the preceding sync_rcu() */
..	..	@@ -3929,8 +4204,7 @@
3929	4204	SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
3930	4205	{
3931	4206	struct load_info info = { };
3932		- loff_t size;
3933		- void *hdr;
	4207	+ void *hdr = NULL;
3934	4208	int err;
3935	4209
3936	4210	err = may_init_module();
..	..	@@ -3943,12 +4217,12 @@
3943	4217	\|MODULE_INIT_IGNORE_VERMAGIC))
3944	4218	return -EINVAL;
3945	4219
3946		- err = kernel_read_file_from_fd(fd, &hdr, &size, INT_MAX,
	4220	+ err = kernel_read_file_from_fd(fd, 0, &hdr, INT_MAX, NULL,
3947	4221	READING_MODULE);
3948		- if (err)
	4222	+ if (err < 0)
3949	4223	return err;
3950	4224	info.hdr = hdr;
3951		- info.len = size;
	4225	+ info.len = err;
3952	4226
3953	4227	return load_module(&info, uargs, flags);
3954	4228	}
..	..	@@ -3971,18 +4245,27 @@
3971	4245	&& (str[2] == '\0' \|\| str[2] == '.');
3972	4246	}
3973	4247
3974		-static const char symname(struct mod_kallsyms kallsyms, unsigned int symnum)
	4248	+static inline int is_cfi_typeid_symbol(const char *str)
	4249	+{
	4250	+ return !strncmp(str, "__typeid__", 10);
	4251	+}
	4252	+
	4253	+static const char kallsyms_symbol_name(struct mod_kallsyms kallsyms, unsigned int symnum)
3975	4254	{
3976	4255	return kallsyms->strtab + kallsyms->symtab[symnum].st_name;
3977	4256	}
3978	4257
3979		-static const char get_ksymbol(struct module mod,
3980		- unsigned long addr,
3981		- unsigned long *size,
3982		- unsigned long *offset)
	4258	+/*
	4259	+ * Given a module and address, find the corresponding symbol and return its name
	4260	+ * while providing its size and offset if needed.
	4261	+ */
	4262	+static const char find_kallsyms_symbol(struct module mod,
	4263	+ unsigned long addr,
	4264	+ unsigned long *size,
	4265	+ unsigned long *offset)
3983	4266	{
3984	4267	unsigned int i, best = 0;
3985		- unsigned long nextval;
	4268	+ unsigned long nextval, bestval;
3986	4269	struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
3987	4270
3988	4271	/* At worse, next value is at end of module */
..	..	@@ -3991,34 +4274,41 @@
3991	4274	else
3992	4275	nextval = (unsigned long)mod->core_layout.base+mod->core_layout.text_size;
3993	4276
	4277	+ bestval = kallsyms_symbol_value(&kallsyms->symtab[best]);
	4278	+
3994	4279	/* Scan for closest preceding symbol, and next symbol. (ELF
3995	4280	starts real symbols at 1). */
3996	4281	for (i = 1; i < kallsyms->num_symtab; i++) {
3997		- if (kallsyms->symtab[i].st_shndx == SHN_UNDEF)
	4282	+ const Elf_Sym *sym = &kallsyms->symtab[i];
	4283	+ unsigned long thisval = kallsyms_symbol_value(sym);
	4284	+
	4285	+ if (sym->st_shndx == SHN_UNDEF)
3998	4286	continue;
3999	4287
4000	4288	/* We ignore unnamed symbols: they're uninformative
4001	4289	* and inserted at a whim. */
4002		- if (*symname(kallsyms, i) == '\0'
4003		- \|\| is_arm_mapping_symbol(symname(kallsyms, i)))
	4290	+ if (*kallsyms_symbol_name(kallsyms, i) == '\0'
	4291	+ \|\| is_arm_mapping_symbol(kallsyms_symbol_name(kallsyms, i))
	4292	+ \|\| is_cfi_typeid_symbol(kallsyms_symbol_name(kallsyms, i)))
4004	4293	continue;
4005	4294
4006		- if (kallsyms->symtab[i].st_value <= addr
4007		- && kallsyms->symtab[i].st_value > kallsyms->symtab[best].st_value)
	4295	+ if (thisval <= addr && thisval > bestval) {
4008	4296	best = i;
4009		- if (kallsyms->symtab[i].st_value > addr
4010		- && kallsyms->symtab[i].st_value < nextval)
4011		- nextval = kallsyms->symtab[i].st_value;
	4297	+ bestval = thisval;
	4298	+ }
	4299	+ if (thisval > addr && thisval < nextval)
	4300	+ nextval = thisval;
4012	4301	}
4013	4302
4014	4303	if (!best)
4015	4304	return NULL;
4016	4305
4017	4306	if (size)
4018		- *size = nextval - kallsyms->symtab[best].st_value;
	4307	+ *size = nextval - bestval;
4019	4308	if (offset)
4020		- *offset = addr - kallsyms->symtab[best].st_value;
4021		- return symname(kallsyms, best);
	4309	+ *offset = addr - bestval;
	4310	+
	4311	+ return kallsyms_symbol_name(kallsyms, best);
4022	4312	}
4023	4313
4024	4314	void * __weak dereference_module_function_descriptor(struct module *mod,
..	..	@@ -4043,7 +4333,8 @@
4043	4333	if (mod) {
4044	4334	if (modname)
4045	4335	*modname = mod->name;
4046		- ret = get_ksymbol(mod, addr, size, offset);
	4336	+
	4337	+ ret = find_kallsyms_symbol(mod, addr, size, offset);
4047	4338	}
4048	4339	/* Make a copy in here where it's safe */
4049	4340	if (ret) {
..	..	@@ -4066,9 +4357,10 @@
4066	4357	if (within_module(addr, mod)) {
4067	4358	const char *sym;
4068	4359
4069		- sym = get_ksymbol(mod, addr, NULL, NULL);
	4360	+ sym = find_kallsyms_symbol(mod, addr, NULL, NULL);
4070	4361	if (!sym)
4071	4362	goto out;
	4363	+
4072	4364	strlcpy(symname, sym, KSYM_NAME_LEN);
4073	4365	preempt_enable();
4074	4366	return 0;
..	..	@@ -4091,7 +4383,7 @@
4091	4383	if (within_module(addr, mod)) {
4092	4384	const char *sym;
4093	4385
4094		- sym = get_ksymbol(mod, addr, size, offset);
	4386	+ sym = find_kallsyms_symbol(mod, addr, size, offset);
4095	4387	if (!sym)
4096	4388	goto out;
4097	4389	if (modname)
..	..	@@ -4120,9 +4412,11 @@
4120	4412	continue;
4121	4413	kallsyms = rcu_dereference_sched(mod->kallsyms);
4122	4414	if (symnum < kallsyms->num_symtab) {
4123		- *value = kallsyms->symtab[symnum].st_value;
4124		- *type = kallsyms->symtab[symnum].st_info;
4125		- strlcpy(name, symname(kallsyms, symnum), KSYM_NAME_LEN);
	4415	+ const Elf_Sym *sym = &kallsyms->symtab[symnum];
	4416	+
	4417	+ *value = kallsyms_symbol_value(sym);
	4418	+ *type = kallsyms->typetab[symnum];
	4419	+ strlcpy(name, kallsyms_symbol_name(kallsyms, symnum), KSYM_NAME_LEN);
4126	4420	strlcpy(module_name, mod->name, MODULE_NAME_LEN);
4127	4421	exported = is_exported(name, value, mod);
4128	4422	preempt_enable();
..	..	@@ -4134,15 +4428,19 @@
4134	4428	return -ERANGE;
4135	4429	}
4136	4430
4137		-static unsigned long mod_find_symname(struct module mod, const char name)
	4431	+/* Given a module and name of symbol, find and return the symbol's value */
	4432	+static unsigned long find_kallsyms_symbol_value(struct module mod, const char name)
4138	4433	{
4139	4434	unsigned int i;
4140	4435	struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
4141	4436
4142		- for (i = 0; i < kallsyms->num_symtab; i++)
4143		- if (strcmp(name, symname(kallsyms, i)) == 0 &&
4144		- kallsyms->symtab[i].st_shndx != SHN_UNDEF)
4145		- return kallsyms->symtab[i].st_value;
	4437	+ for (i = 0; i < kallsyms->num_symtab; i++) {
	4438	+ const Elf_Sym *sym = &kallsyms->symtab[i];
	4439	+
	4440	+ if (strcmp(name, kallsyms_symbol_name(kallsyms, i)) == 0 &&
	4441	+ sym->st_shndx != SHN_UNDEF)
	4442	+ return kallsyms_symbol_value(sym);
	4443	+ }
4146	4444	return 0;
4147	4445	}
4148	4446
..	..	@@ -4157,12 +4455,12 @@
4157	4455	preempt_disable();
4158	4456	if ((colon = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) {
4159	4457	if ((mod = find_module_all(name, colon - name, false)) != NULL)
4160		- ret = mod_find_symname(mod, colon+1);
	4458	+ ret = find_kallsyms_symbol_value(mod, colon+1);
4161	4459	} else {
4162	4460	list_for_each_entry_rcu(mod, &modules, list) {
4163	4461	if (mod->state == MODULE_STATE_UNFORMED)
4164	4462	continue;
4165		- if ((ret = mod_find_symname(mod, name)) != 0)
	4463	+ if ((ret = find_kallsyms_symbol_value(mod, name)) != 0)
4166	4464	break;
4167	4465	}
4168	4466	}
..	..	@@ -4187,12 +4485,13 @@
4187	4485	if (mod->state == MODULE_STATE_UNFORMED)
4188	4486	continue;
4189	4487	for (i = 0; i < kallsyms->num_symtab; i++) {
	4488	+ const Elf_Sym *sym = &kallsyms->symtab[i];
4190	4489
4191		- if (kallsyms->symtab[i].st_shndx == SHN_UNDEF)
	4490	+ if (sym->st_shndx == SHN_UNDEF)
4192	4491	continue;
4193	4492
4194		- ret = fn(data, symname(kallsyms, i),
4195		- mod, kallsyms->symtab[i].st_value);
	4493	+ ret = fn(data, kallsyms_symbol_name(kallsyms, i),
	4494	+ mod, kallsyms_symbol_value(sym));
4196	4495	if (ret != 0)
4197	4496	return ret;
4198	4497	}
..	..	@@ -4204,18 +4503,30 @@
4204	4503	static void cfi_init(struct module *mod)
4205	4504	{
4206	4505	#ifdef CONFIG_CFI_CLANG
4207		- preempt_disable();
4208		- mod->cfi_check =
4209		- (cfi_check_fn)mod_find_symname(mod, CFI_CHECK_FN_NAME);
4210		- preempt_enable();
4211		- cfi_module_add(mod, module_addr_min, module_addr_max);
	4506	+ initcall_t *init;
	4507	+ exitcall_t *exit;
	4508	+
	4509	+ rcu_read_lock_sched();
	4510	+ mod->cfi_check = (cfi_check_fn)
	4511	+ find_kallsyms_symbol_value(mod, "__cfi_check");
	4512	+ init = (initcall_t *)
	4513	+ find_kallsyms_symbol_value(mod, "__cfi_jt_init_module");
	4514	+ exit = (exitcall_t *)
	4515	+ find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module");
	4516	+ rcu_read_unlock_sched();
	4517	+
	4518	+ /* Fix init/exit functions to point to the CFI jump table */
	4519	+ if (init) mod->init = *init;
	4520	+ if (exit) mod->exit = *exit;
	4521	+
	4522	+ cfi_module_add(mod, module_addr_min);
4212	4523	#endif
4213	4524	}
4214	4525
4215	4526	static void cfi_cleanup(struct module *mod)
4216	4527	{
4217	4528	#ifdef CONFIG_CFI_CLANG
4218		- cfi_module_remove(mod, module_addr_min, module_addr_max);
	4529	+ cfi_module_remove(mod, module_addr_min);
4219	4530	#endif
4220	4531	}
4221	4532
..	..	@@ -4326,16 +4637,17 @@
4326	4637	return err;
4327	4638	}
4328	4639
4329		-static const struct file_operations proc_modules_operations = {
4330		- .open = modules_open,
4331		- .read = seq_read,
4332		- .llseek = seq_lseek,
4333		- .release = seq_release,
	4640	+static const struct proc_ops modules_proc_ops = {
	4641	+ .proc_flags = PROC_ENTRY_PERMANENT,
	4642	+ .proc_open = modules_open,
	4643	+ .proc_read = seq_read,
	4644	+ .proc_lseek = seq_lseek,
	4645	+ .proc_release = seq_release,
4334	4646	};
4335	4647
4336	4648	static int __init proc_modules_init(void)
4337	4649	{
4338		- proc_create("modules", 0, NULL, &proc_modules_operations);
	4650	+ proc_create("modules", 0, NULL, &modules_proc_ops);
4339	4651	return 0;
4340	4652	}
4341	4653	module_init(proc_modules_init);
..	..	@@ -4469,6 +4781,23 @@
4469	4781	pr_cont("\n");
4470	4782	}
4471	4783
	4784	+#ifdef CONFIG_ANDROID_DEBUG_SYMBOLS
	4785	+void android_debug_for_each_module(int (fn)(const char mod_name, void mod_addr, void data),
	4786	+ void *data)
	4787	+{
	4788	+ struct module *module;
	4789	+
	4790	+ preempt_disable();
	4791	+ list_for_each_entry_rcu(module, &modules, list) {
	4792	+ if (fn(module->name, module->core_layout.base, data))
	4793	+ goto out;
	4794	+ }
	4795	+out:
	4796	+ preempt_enable();
	4797	+}
	4798	+EXPORT_SYMBOL_GPL(android_debug_for_each_module);
	4799	+#endif
	4800	+
4472	4801	#ifdef CONFIG_MODVERSIONS
4473	4802	/* Generate the signature for all relevant module structures here.
4474	4803	* If these change, we don't want to try to parse the module. */