~hc/RK356X_SDK_RELEASE.git

..	..	@@ -17,6 +17,7 @@
17	17	#include <linux/stacktrace.h>
18	18	#include <linux/writeback.h>
19	19	#include <linux/kallsyms.h>
	20	+#include <linux/security.h>
20	21	#include <linux/seq_file.h>
21	22	#include <linux/notifier.h>
22	23	#include <linux/irqflags.h>
..	..	@@ -44,6 +45,10 @@
44	45	#include <linux/trace.h>
45	46	#include <linux/sched/clock.h>
46	47	#include <linux/sched/rt.h>
	48	+#include <linux/fsnotify.h>
	49	+#include <linux/irq_work.h>
	50	+#include <linux/workqueue.h>
	51	+#include <trace/hooks/ftrace_dump.h>
47	52
48	53	#include "trace.h"
49	54	#include "trace_output.h"
..	..	@@ -64,9 +69,20 @@
64	69	static bool __read_mostly tracing_selftest_running;
65	70
66	71	/*
67		- * If a tracer is running, we do not want to run SELFTEST.
	72	+ * If boot-time tracing including tracers/events via kernel cmdline
	73	+ * is running, we do not want to run SELFTEST.
68	74	*/
69	75	bool __read_mostly tracing_selftest_disabled;
	76	+
	77	+#ifdef CONFIG_FTRACE_STARTUP_TEST
	78	+void __init disable_tracing_selftest(const char *reason)
	79	+{
	80	+ if (!tracing_selftest_disabled) {
	81	+ tracing_selftest_disabled = true;
	82	+ pr_info("Ftrace startup test is disabled due to %s\n", reason);
	83	+ }
	84	+}
	85	+#endif
70	86
71	87	/* Pipe tracepoints to printk */
72	88	struct trace_iterator *tracepoint_print_iter;
..	..	@@ -158,7 +174,10 @@
158	174	static union trace_eval_map_item *trace_eval_maps;
159	175	#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
160	176
161		-static int tracing_set_tracer(struct trace_array tr, const char buf);
	177	+int tracing_set_tracer(struct trace_array tr, const char buf);
	178	+static void ftrace_trace_userstack(struct trace_array *tr,
	179	+ struct trace_buffer *buffer,
	180	+ unsigned int trace_ctx);
162	181
163	182	#define MAX_TRACER_SIZE 100
164	183	static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
..	..	@@ -215,7 +234,7 @@
215	234	static int __init set_trace_boot_options(char *str)
216	235	{
217	236	strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
218		- return 0;
	237	+ return 1;
219	238	}
220	239	__setup("trace_options=", set_trace_boot_options);
221	240
..	..	@@ -226,7 +245,7 @@
226	245	{
227	246	strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
228	247	trace_boot_clock = trace_boot_clock_buf;
229		- return 0;
	248	+ return 1;
230	249	}
231	250	__setup("trace_clock=", set_trace_boot_clock);
232	251
..	..	@@ -248,6 +267,145 @@
248	267	do_div(nsec, 1000);
249	268	return nsec;
250	269	}
	270	+
	271	+static void
	272	+trace_process_export(struct trace_export *export,
	273	+ struct ring_buffer_event *event, int flag)
	274	+{
	275	+ struct trace_entry *entry;
	276	+ unsigned int size = 0;
	277	+
	278	+ if (export->flags & flag) {
	279	+ entry = ring_buffer_event_data(event);
	280	+ size = ring_buffer_event_length(event);
	281	+ export->write(export, entry, size);
	282	+ }
	283	+}
	284	+
	285	+static DEFINE_MUTEX(ftrace_export_lock);
	286	+
	287	+static struct trace_export __rcu *ftrace_exports_list __read_mostly;
	288	+
	289	+static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
	290	+static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
	291	+static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
	292	+
	293	+static inline void ftrace_exports_enable(struct trace_export *export)
	294	+{
	295	+ if (export->flags & TRACE_EXPORT_FUNCTION)
	296	+ static_branch_inc(&trace_function_exports_enabled);
	297	+
	298	+ if (export->flags & TRACE_EXPORT_EVENT)
	299	+ static_branch_inc(&trace_event_exports_enabled);
	300	+
	301	+ if (export->flags & TRACE_EXPORT_MARKER)
	302	+ static_branch_inc(&trace_marker_exports_enabled);
	303	+}
	304	+
	305	+static inline void ftrace_exports_disable(struct trace_export *export)
	306	+{
	307	+ if (export->flags & TRACE_EXPORT_FUNCTION)
	308	+ static_branch_dec(&trace_function_exports_enabled);
	309	+
	310	+ if (export->flags & TRACE_EXPORT_EVENT)
	311	+ static_branch_dec(&trace_event_exports_enabled);
	312	+
	313	+ if (export->flags & TRACE_EXPORT_MARKER)
	314	+ static_branch_dec(&trace_marker_exports_enabled);
	315	+}
	316	+
	317	+static void ftrace_exports(struct ring_buffer_event *event, int flag)
	318	+{
	319	+ struct trace_export *export;
	320	+
	321	+ preempt_disable_notrace();
	322	+
	323	+ export = rcu_dereference_raw_check(ftrace_exports_list);
	324	+ while (export) {
	325	+ trace_process_export(export, event, flag);
	326	+ export = rcu_dereference_raw_check(export->next);
	327	+ }
	328	+
	329	+ preempt_enable_notrace();
	330	+}
	331	+
	332	+static inline void
	333	+add_trace_export(struct trace_export *list, struct trace_export export)
	334	+{
	335	+ rcu_assign_pointer(export->next, *list);
	336	+ /*
	337	+ * We are entering export into the list but another
	338	+ * CPU might be walking that list. We need to make sure
	339	+ * the export->next pointer is valid before another CPU sees
	340	+ * the export pointer included into the list.
	341	+ */
	342	+ rcu_assign_pointer(*list, export);
	343	+}
	344	+
	345	+static inline int
	346	+rm_trace_export(struct trace_export *list, struct trace_export export)
	347	+{
	348	+ struct trace_export **p;
	349	+
	350	+ for (p = list; p != NULL; p = &(p)->next)
	351	+ if (*p == export)
	352	+ break;
	353	+
	354	+ if (*p != export)
	355	+ return -1;
	356	+
	357	+ rcu_assign_pointer(p, (p)->next);
	358	+
	359	+ return 0;
	360	+}
	361	+
	362	+static inline void
	363	+add_ftrace_export(struct trace_export *list, struct trace_export export)
	364	+{
	365	+ ftrace_exports_enable(export);
	366	+
	367	+ add_trace_export(list, export);
	368	+}
	369	+
	370	+static inline int
	371	+rm_ftrace_export(struct trace_export *list, struct trace_export export)
	372	+{
	373	+ int ret;
	374	+
	375	+ ret = rm_trace_export(list, export);
	376	+ ftrace_exports_disable(export);
	377	+
	378	+ return ret;
	379	+}
	380	+
	381	+int register_ftrace_export(struct trace_export *export)
	382	+{
	383	+ if (WARN_ON_ONCE(!export->write))
	384	+ return -1;
	385	+
	386	+ mutex_lock(&ftrace_export_lock);
	387	+
	388	+ add_ftrace_export(&ftrace_exports_list, export);
	389	+
	390	+ mutex_unlock(&ftrace_export_lock);
	391	+
	392	+ return 0;
	393	+}
	394	+EXPORT_SYMBOL_GPL(register_ftrace_export);
	395	+
	396	+int unregister_ftrace_export(struct trace_export *export)
	397	+{
	398	+ int ret;
	399	+
	400	+ mutex_lock(&ftrace_export_lock);
	401	+
	402	+ ret = rm_ftrace_export(&ftrace_exports_list, export);
	403	+
	404	+ mutex_unlock(&ftrace_export_lock);
	405	+
	406	+ return ret;
	407	+}
	408	+EXPORT_SYMBOL_GPL(unregister_ftrace_export);
251	409
252	410	/* trace_flags holds trace_options default values */
253	411	#define TRACE_DEFAULT_FLAGS \
..	..	@@ -299,15 +457,44 @@
299	457	this_tr->ref--;
300	458	}
301	459
	460	+/**
	461	+ * trace_array_put - Decrement the reference counter for this trace array.
	462	+ *
	463	+ * NOTE: Use this when we no longer need the trace array returned by
	464	+ * trace_array_get_by_name(). This ensures the trace array can be later
	465	+ * destroyed.
	466	+ *
	467	+ */
302	468	void trace_array_put(struct trace_array *this_tr)
303	469	{
	470	+ if (!this_tr)
	471	+ return;
	472	+
304	473	mutex_lock(&trace_types_lock);
305	474	__trace_array_put(this_tr);
306	475	mutex_unlock(&trace_types_lock);
307	476	}
	477	+EXPORT_SYMBOL_GPL(trace_array_put);
	478	+
	479	+int tracing_check_open_get_tr(struct trace_array *tr)
	480	+{
	481	+ int ret;
	482	+
	483	+ ret = security_locked_down(LOCKDOWN_TRACEFS);
	484	+ if (ret)
	485	+ return ret;
	486	+
	487	+ if (tracing_disabled)
	488	+ return -ENODEV;
	489	+
	490	+ if (tr && trace_array_get(tr) < 0)
	491	+ return -ENODEV;
	492	+
	493	+ return 0;
	494	+}
308	495
309	496	int call_filter_check_discard(struct trace_event_call call, void rec,
310		- struct ring_buffer *buffer,
	497	+ struct trace_buffer *buffer,
311	498	struct ring_buffer_event *event)
312	499	{
313	500	if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
..	..	@@ -355,20 +542,26 @@
355	542	* Returns false if @task should be traced.
356	543	*/
357	544	bool
358		-trace_ignore_this_task(struct trace_pid_list filtered_pids, struct task_struct task)
	545	+trace_ignore_this_task(struct trace_pid_list *filtered_pids,
	546	+ struct trace_pid_list *filtered_no_pids,
	547	+ struct task_struct *task)
359	548	{
360	549	/*
361		- * Return false, because if filtered_pids does not exist,
362		- * all pids are good to trace.
	550	+ * If filterd_no_pids is not empty, and the task's pid is listed
	551	+ * in filtered_no_pids, then return true.
	552	+ * Otherwise, if filtered_pids is empty, that means we can
	553	+ * trace all tasks. If it has content, then only trace pids
	554	+ * within filtered_pids.
363	555	*/
364		- if (!filtered_pids)
365		- return false;
366	556
367		- return !trace_find_filtered_pid(filtered_pids, task->pid);
	557	+ return (filtered_pids &&
	558	+ !trace_find_filtered_pid(filtered_pids, task->pid)) \|\|
	559	+ (filtered_no_pids &&
	560	+ trace_find_filtered_pid(filtered_no_pids, task->pid));
368	561	}
369	562
370	563	/**
371		- * trace_pid_filter_add_remove_task - Add or remove a task from a pid_list
	564	+ * trace_filter_add_remove_task - Add or remove a task from a pid_list
372	565	* @pid_list: The list to modify
373	566	* @self: The current task for fork or NULL for exit
374	567	* @task: The task to add or remove
..	..	@@ -572,7 +765,7 @@
572	765	return read;
573	766	}
574	767
575		-static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu)
	768	+static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
576	769	{
577	770	u64 ts;
578	771
..	..	@@ -588,7 +781,7 @@
588	781
589	782	u64 ftrace_now(int cpu)
590	783	{
591		- return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
	784	+ return buffer_ftrace_now(&global_trace.array_buffer, cpu);
592	785	}
593	786
594	787	/**
..	..	@@ -716,24 +909,24 @@
716	909	#endif
717	910
718	911	#ifdef CONFIG_STACKTRACE
719		-static void __ftrace_trace_stack(struct ring_buffer *buffer,
720		- unsigned long flags,
721		- int skip, int pc, struct pt_regs *regs);
	912	+static void __ftrace_trace_stack(struct trace_buffer *buffer,
	913	+ unsigned int trace_ctx,
	914	+ int skip, struct pt_regs *regs);
722	915	static inline void ftrace_trace_stack(struct trace_array *tr,
723		- struct ring_buffer *buffer,
724		- unsigned long flags,
725		- int skip, int pc, struct pt_regs *regs);
	916	+ struct trace_buffer *buffer,
	917	+ unsigned int trace_ctx,
	918	+ int skip, struct pt_regs *regs);
726	919
727	920	#else
728		-static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
729		- unsigned long flags,
730		- int skip, int pc, struct pt_regs *regs)
	921	+static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
	922	+ unsigned int trace_ctx,
	923	+ int skip, struct pt_regs *regs)
731	924	{
732	925	}
733	926	static inline void ftrace_trace_stack(struct trace_array *tr,
734		- struct ring_buffer *buffer,
735		- unsigned long flags,
736		- int skip, int pc, struct pt_regs *regs)
	927	+ struct trace_buffer *buffer,
	928	+ unsigned long trace_ctx,
	929	+ int skip, struct pt_regs *regs)
737	930	{
738	931	}
739	932
..	..	@@ -741,33 +934,32 @@
741	934
742	935	static __always_inline void
743	936	trace_event_setup(struct ring_buffer_event *event,
744		- int type, unsigned long flags, int pc)
	937	+ int type, unsigned int trace_ctx)
745	938	{
746	939	struct trace_entry *ent = ring_buffer_event_data(event);
747	940
748		- tracing_generic_entry_update(ent, flags, pc);
749		- ent->type = type;
	941	+ tracing_generic_entry_update(ent, type, trace_ctx);
750	942	}
751	943
752	944	static __always_inline struct ring_buffer_event *
753		-__trace_buffer_lock_reserve(struct ring_buffer *buffer,
	945	+__trace_buffer_lock_reserve(struct trace_buffer *buffer,
754	946	int type,
755	947	unsigned long len,
756		- unsigned long flags, int pc)
	948	+ unsigned int trace_ctx)
757	949	{
758	950	struct ring_buffer_event *event;
759	951
760	952	event = ring_buffer_lock_reserve(buffer, len);
761	953	if (event != NULL)
762		- trace_event_setup(event, type, flags, pc);
	954	+ trace_event_setup(event, type, trace_ctx);
763	955
764	956	return event;
765	957	}
766	958
767	959	void tracer_tracing_on(struct trace_array *tr)
768	960	{
769		- if (tr->trace_buffer.buffer)
770		- ring_buffer_record_on(tr->trace_buffer.buffer);
	961	+ if (tr->array_buffer.buffer)
	962	+ ring_buffer_record_on(tr->array_buffer.buffer);
771	963	/*
772	964	* This flag is looked at when buffers haven't been allocated
773	965	* yet, or by some tracers (like irqsoff), that just want to
..	..	@@ -795,7 +987,7 @@
795	987
796	988
797	989	static __always_inline void
798		-__buffer_unlock_commit(struct ring_buffer buffer, struct ring_buffer_event event)
	990	+__buffer_unlock_commit(struct trace_buffer buffer, struct ring_buffer_event event)
799	991	{
800	992	__this_cpu_write(trace_taskinfo_save, true);
801	993
..	..	@@ -818,28 +1010,28 @@
818	1010	int __trace_puts(unsigned long ip, const char *str, int size)
819	1011	{
820	1012	struct ring_buffer_event *event;
821		- struct ring_buffer *buffer;
	1013	+ struct trace_buffer *buffer;
822	1014	struct print_entry *entry;
823		- unsigned long irq_flags;
	1015	+ unsigned int trace_ctx;
824	1016	int alloc;
825		- int pc;
826	1017
827	1018	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
828	1019	return 0;
829		-
830		- pc = preempt_count();
831	1020
832	1021	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
833	1022	return 0;
834	1023
835	1024	alloc = sizeof(entry) + size + 2; / possible \n added */
836	1025
837		- local_save_flags(irq_flags);
838		- buffer = global_trace.trace_buffer.buffer;
839		- event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
840		- irq_flags, pc);
841		- if (!event)
842		- return 0;
	1026	+ trace_ctx = tracing_gen_ctx();
	1027	+ buffer = global_trace.array_buffer.buffer;
	1028	+ ring_buffer_nest_start(buffer);
	1029	+ event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
	1030	+ trace_ctx);
	1031	+ if (!event) {
	1032	+ size = 0;
	1033	+ goto out;
	1034	+ }
843	1035
844	1036	entry = ring_buffer_event_data(event);
845	1037	entry->ip = ip;
..	..	@@ -854,8 +1046,9 @@
854	1046	entry->buf[size] = '\0';
855	1047
856	1048	__buffer_unlock_commit(buffer, event);
857		- ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
858		-
	1049	+ ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);
	1050	+ out:
	1051	+ ring_buffer_nest_end(buffer);
859	1052	return size;
860	1053	}
861	1054	EXPORT_SYMBOL_GPL(__trace_puts);
..	..	@@ -868,40 +1061,44 @@
868	1061	int __trace_bputs(unsigned long ip, const char *str)
869	1062	{
870	1063	struct ring_buffer_event *event;
871		- struct ring_buffer *buffer;
	1064	+ struct trace_buffer *buffer;
872	1065	struct bputs_entry *entry;
873		- unsigned long irq_flags;
	1066	+ unsigned int trace_ctx;
874	1067	int size = sizeof(struct bputs_entry);
875		- int pc;
	1068	+ int ret = 0;
876	1069
877	1070	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
878	1071	return 0;
879	1072
880		- pc = preempt_count();
881		-
882	1073	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
883	1074	return 0;
884	1075
885		- local_save_flags(irq_flags);
886		- buffer = global_trace.trace_buffer.buffer;
	1076	+ trace_ctx = tracing_gen_ctx();
	1077	+ buffer = global_trace.array_buffer.buffer;
	1078	+
	1079	+ ring_buffer_nest_start(buffer);
887	1080	event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
888		- irq_flags, pc);
	1081	+ trace_ctx);
889	1082	if (!event)
890		- return 0;
	1083	+ goto out;
891	1084
892	1085	entry = ring_buffer_event_data(event);
893	1086	entry->ip = ip;
894	1087	entry->str = str;
895	1088
896	1089	__buffer_unlock_commit(buffer, event);
897		- ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
	1090	+ ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);
898	1091
899		- return 1;
	1092	+ ret = 1;
	1093	+ out:
	1094	+ ring_buffer_nest_end(buffer);
	1095	+ return ret;
900	1096	}
901	1097	EXPORT_SYMBOL_GPL(__trace_bputs);
902	1098
903	1099	#ifdef CONFIG_TRACER_SNAPSHOT
904		-void tracing_snapshot_instance(struct trace_array *tr)
	1100	+static void tracing_snapshot_instance_cond(struct trace_array *tr,
	1101	+ void *cond_data)
905	1102	{
906	1103	struct tracer *tracer = tr->current_trace;
907	1104	unsigned long flags;
..	..	@@ -927,8 +1124,13 @@
927	1124	}
928	1125
929	1126	local_irq_save(flags);
930		- update_max_tr(tr, current, smp_processor_id());
	1127	+ update_max_tr(tr, current, smp_processor_id(), cond_data);
931	1128	local_irq_restore(flags);
	1129	+}
	1130	+
	1131	+void tracing_snapshot_instance(struct trace_array *tr)
	1132	+{
	1133	+ tracing_snapshot_instance_cond(tr, NULL);
932	1134	}
933	1135
934	1136	/**
..	..	@@ -953,9 +1155,59 @@
953	1155	}
954	1156	EXPORT_SYMBOL_GPL(tracing_snapshot);
955	1157
956		-static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
957		- struct trace_buffer *size_buf, int cpu_id);
958		-static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
	1158	+/**
	1159	+ * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
	1160	+ * @tr: The tracing instance to snapshot
	1161	+ * @cond_data: The data to be tested conditionally, and possibly saved
	1162	+ *
	1163	+ * This is the same as tracing_snapshot() except that the snapshot is
	1164	+ * conditional - the snapshot will only happen if the
	1165	+ * cond_snapshot.update() implementation receiving the cond_data
	1166	+ * returns true, which means that the trace array's cond_snapshot
	1167	+ * update() operation used the cond_data to determine whether the
	1168	+ * snapshot should be taken, and if it was, presumably saved it along
	1169	+ * with the snapshot.
	1170	+ */
	1171	+void tracing_snapshot_cond(struct trace_array tr, void cond_data)
	1172	+{
	1173	+ tracing_snapshot_instance_cond(tr, cond_data);
	1174	+}
	1175	+EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
	1176	+
	1177	+/**
	1178	+ * tracing_snapshot_cond_data - get the user data associated with a snapshot
	1179	+ * @tr: The tracing instance
	1180	+ *
	1181	+ * When the user enables a conditional snapshot using
	1182	+ * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
	1183	+ * with the snapshot. This accessor is used to retrieve it.
	1184	+ *
	1185	+ * Should not be called from cond_snapshot.update(), since it takes
	1186	+ * the tr->max_lock lock, which the code calling
	1187	+ * cond_snapshot.update() has already done.
	1188	+ *
	1189	+ * Returns the cond_data associated with the trace array's snapshot.
	1190	+ */
	1191	+void tracing_cond_snapshot_data(struct trace_array tr)
	1192	+{
	1193	+ void *cond_data = NULL;
	1194	+
	1195	+ local_irq_disable();
	1196	+ arch_spin_lock(&tr->max_lock);
	1197	+
	1198	+ if (tr->cond_snapshot)
	1199	+ cond_data = tr->cond_snapshot->cond_data;
	1200	+
	1201	+ arch_spin_unlock(&tr->max_lock);
	1202	+ local_irq_enable();
	1203	+
	1204	+ return cond_data;
	1205	+}
	1206	+EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
	1207	+
	1208	+static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
	1209	+ struct array_buffer *size_buf, int cpu_id);
	1210	+static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
959	1211
960	1212	int tracing_alloc_snapshot_instance(struct trace_array *tr)
961	1213	{
..	..	@@ -965,7 +1217,7 @@
965	1217
966	1218	/* allocate spare buffer */
967	1219	ret = resize_buffer_duplicate_size(&tr->max_buffer,
968		- &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
	1220	+ &tr->array_buffer, RING_BUFFER_ALL_CPUS);
969	1221	if (ret < 0)
970	1222	return ret;
971	1223
..	..	@@ -1032,12 +1284,115 @@
1032	1284	tracing_snapshot();
1033	1285	}
1034	1286	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
	1287	+
	1288	+/**
	1289	+ * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
	1290	+ * @tr: The tracing instance
	1291	+ * @cond_data: User data to associate with the snapshot
	1292	+ * @update: Implementation of the cond_snapshot update function
	1293	+ *
	1294	+ * Check whether the conditional snapshot for the given instance has
	1295	+ * already been enabled, or if the current tracer is already using a
	1296	+ * snapshot; if so, return -EBUSY, else create a cond_snapshot and
	1297	+ * save the cond_data and update function inside.
	1298	+ *
	1299	+ * Returns 0 if successful, error otherwise.
	1300	+ */
	1301	+int tracing_snapshot_cond_enable(struct trace_array tr, void cond_data,
	1302	+ cond_update_fn_t update)
	1303	+{
	1304	+ struct cond_snapshot *cond_snapshot;
	1305	+ int ret = 0;
	1306	+
	1307	+ cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
	1308	+ if (!cond_snapshot)
	1309	+ return -ENOMEM;
	1310	+
	1311	+ cond_snapshot->cond_data = cond_data;
	1312	+ cond_snapshot->update = update;
	1313	+
	1314	+ mutex_lock(&trace_types_lock);
	1315	+
	1316	+ ret = tracing_alloc_snapshot_instance(tr);
	1317	+ if (ret)
	1318	+ goto fail_unlock;
	1319	+
	1320	+ if (tr->current_trace->use_max_tr) {
	1321	+ ret = -EBUSY;
	1322	+ goto fail_unlock;
	1323	+ }
	1324	+
	1325	+ /*
	1326	+ * The cond_snapshot can only change to NULL without the
	1327	+ * trace_types_lock. We don't care if we race with it going
	1328	+ * to NULL, but we want to make sure that it's not set to
	1329	+ * something other than NULL when we get here, which we can
	1330	+ * do safely with only holding the trace_types_lock and not
	1331	+ * having to take the max_lock.
	1332	+ */
	1333	+ if (tr->cond_snapshot) {
	1334	+ ret = -EBUSY;
	1335	+ goto fail_unlock;
	1336	+ }
	1337	+
	1338	+ local_irq_disable();
	1339	+ arch_spin_lock(&tr->max_lock);
	1340	+ tr->cond_snapshot = cond_snapshot;
	1341	+ arch_spin_unlock(&tr->max_lock);
	1342	+ local_irq_enable();
	1343	+
	1344	+ mutex_unlock(&trace_types_lock);
	1345	+
	1346	+ return ret;
	1347	+
	1348	+ fail_unlock:
	1349	+ mutex_unlock(&trace_types_lock);
	1350	+ kfree(cond_snapshot);
	1351	+ return ret;
	1352	+}
	1353	+EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
	1354	+
	1355	+/**
	1356	+ * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
	1357	+ * @tr: The tracing instance
	1358	+ *
	1359	+ * Check whether the conditional snapshot for the given instance is
	1360	+ * enabled; if so, free the cond_snapshot associated with it,
	1361	+ * otherwise return -EINVAL.
	1362	+ *
	1363	+ * Returns 0 if successful, error otherwise.
	1364	+ */
	1365	+int tracing_snapshot_cond_disable(struct trace_array *tr)
	1366	+{
	1367	+ int ret = 0;
	1368	+
	1369	+ local_irq_disable();
	1370	+ arch_spin_lock(&tr->max_lock);
	1371	+
	1372	+ if (!tr->cond_snapshot)
	1373	+ ret = -EINVAL;
	1374	+ else {
	1375	+ kfree(tr->cond_snapshot);
	1376	+ tr->cond_snapshot = NULL;
	1377	+ }
	1378	+
	1379	+ arch_spin_unlock(&tr->max_lock);
	1380	+ local_irq_enable();
	1381	+
	1382	+ return ret;
	1383	+}
	1384	+EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1035	1385	#else
1036	1386	void tracing_snapshot(void)
1037	1387	{
1038	1388	WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1039	1389	}
1040	1390	EXPORT_SYMBOL_GPL(tracing_snapshot);
	1391	+void tracing_snapshot_cond(struct trace_array tr, void cond_data)
	1392	+{
	1393	+ WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
	1394	+}
	1395	+EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1041	1396	int tracing_alloc_snapshot(void)
1042	1397	{
1043	1398	WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
..	..	@@ -1050,12 +1405,27 @@
1050	1405	tracing_snapshot();
1051	1406	}
1052	1407	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
	1408	+void tracing_cond_snapshot_data(struct trace_array tr)
	1409	+{
	1410	+ return NULL;
	1411	+}
	1412	+EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
	1413	+int tracing_snapshot_cond_enable(struct trace_array tr, void cond_data, cond_update_fn_t update)
	1414	+{
	1415	+ return -ENODEV;
	1416	+}
	1417	+EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
	1418	+int tracing_snapshot_cond_disable(struct trace_array *tr)
	1419	+{
	1420	+ return false;
	1421	+}
	1422	+EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1053	1423	#endif /* CONFIG_TRACER_SNAPSHOT */
1054	1424
1055	1425	void tracer_tracing_off(struct trace_array *tr)
1056	1426	{
1057		- if (tr->trace_buffer.buffer)
1058		- ring_buffer_record_off(tr->trace_buffer.buffer);
	1427	+ if (tr->array_buffer.buffer)
	1428	+ ring_buffer_record_off(tr->array_buffer.buffer);
1059	1429	/*
1060	1430	* This flag is looked at when buffers haven't been allocated
1061	1431	* yet, or by some tracers (like irqsoff), that just want to
..	..	@@ -1085,8 +1455,11 @@
1085	1455
1086	1456	void disable_trace_on_warning(void)
1087	1457	{
1088		- if (__disable_trace_on_warning)
	1458	+ if (__disable_trace_on_warning) {
	1459	+ trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
	1460	+ "Disabling tracing due to warning\n");
1089	1461	tracing_off();
	1462	+ }
1090	1463	}
1091	1464
1092	1465	/**
..	..	@@ -1097,8 +1470,8 @@
1097	1470	*/
1098	1471	bool tracer_tracing_is_on(struct trace_array *tr)
1099	1472	{
1100		- if (tr->trace_buffer.buffer)
1101		- return ring_buffer_record_is_on(tr->trace_buffer.buffer);
	1473	+ if (tr->array_buffer.buffer)
	1474	+ return ring_buffer_record_is_on(tr->array_buffer.buffer);
1102	1475	return !tr->buffer_disabled;
1103	1476	}
1104	1477
..	..	@@ -1118,10 +1491,12 @@
1118	1491	if (!str)
1119	1492	return 0;
1120	1493	buf_size = memparse(str, &str);
1121		- /* nr_entries can not be zero */
1122		- if (buf_size == 0)
1123		- return 0;
1124		- trace_buf_size = buf_size;
	1494	+ /*
	1495	+ * nr_entries can not be zero and the startup
	1496	+ * tests require some buffer space. Therefore
	1497	+ * ensure we have at least 4096 bytes of buffer.
	1498	+ */
	1499	+ trace_buf_size = max(4096UL, buf_size);
1125	1500	return 1;
1126	1501	}
1127	1502	__setup("trace_buf_size=", set_buf_size);
..	..	@@ -1315,6 +1690,73 @@
1315	1690	}
1316	1691
1317	1692	unsigned long __read_mostly tracing_thresh;
	1693	+static const struct file_operations tracing_max_lat_fops;
	1694	+
	1695	+#if (defined(CONFIG_TRACER_MAX_TRACE) \|\| defined(CONFIG_HWLAT_TRACER)) && \
	1696	+ defined(CONFIG_FSNOTIFY)
	1697	+
	1698	+static struct workqueue_struct *fsnotify_wq;
	1699	+
	1700	+static void latency_fsnotify_workfn(struct work_struct *work)
	1701	+{
	1702	+ struct trace_array *tr = container_of(work, struct trace_array,
	1703	+ fsnotify_work);
	1704	+ fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);
	1705	+}
	1706	+
	1707	+static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
	1708	+{
	1709	+ struct trace_array *tr = container_of(iwork, struct trace_array,
	1710	+ fsnotify_irqwork);
	1711	+ queue_work(fsnotify_wq, &tr->fsnotify_work);
	1712	+}
	1713	+
	1714	+static void trace_create_maxlat_file(struct trace_array *tr,
	1715	+ struct dentry *d_tracer)
	1716	+{
	1717	+ INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
	1718	+ init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
	1719	+ tr->d_max_latency = trace_create_file("tracing_max_latency", 0644,
	1720	+ d_tracer, &tr->max_latency,
	1721	+ &tracing_max_lat_fops);
	1722	+}
	1723	+
	1724	+__init static int latency_fsnotify_init(void)
	1725	+{
	1726	+ fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
	1727	+ WQ_UNBOUND \| WQ_HIGHPRI, 0);
	1728	+ if (!fsnotify_wq) {
	1729	+ pr_err("Unable to allocate tr_max_lat_wq\n");
	1730	+ return -ENOMEM;
	1731	+ }
	1732	+ return 0;
	1733	+}
	1734	+
	1735	+late_initcall_sync(latency_fsnotify_init);
	1736	+
	1737	+void latency_fsnotify(struct trace_array *tr)
	1738	+{
	1739	+ if (!fsnotify_wq)
	1740	+ return;
	1741	+ /*
	1742	+ * We cannot call queue_work(&tr->fsnotify_work) from here because it's
	1743	+ * possible that we are called from __schedule() or do_idle(), which
	1744	+ * could cause a deadlock.
	1745	+ */
	1746	+ irq_work_queue(&tr->fsnotify_irqwork);
	1747	+}
	1748	+
	1749	+/*
	1750	+ * (defined(CONFIG_TRACER_MAX_TRACE) \|\| defined(CONFIG_HWLAT_TRACER)) && \
	1751	+ * defined(CONFIG_FSNOTIFY)
	1752	+ */
	1753	+#else
	1754	+
	1755	+#define trace_create_maxlat_file(tr, d_tracer) \
	1756	+ trace_create_file("tracing_max_latency", 0644, d_tracer, \
	1757	+ &tr->max_latency, &tracing_max_lat_fops)
	1758	+
	1759	+#endif
1318	1760
1319	1761	#ifdef CONFIG_TRACER_MAX_TRACE
1320	1762	/*
..	..	@@ -1325,8 +1767,8 @@
1325	1767	static void
1326	1768	__update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
1327	1769	{
1328		- struct trace_buffer *trace_buf = &tr->trace_buffer;
1329		- struct trace_buffer *max_buf = &tr->max_buffer;
	1770	+ struct array_buffer *trace_buf = &tr->array_buffer;
	1771	+ struct array_buffer *max_buf = &tr->max_buffer;
1330	1772	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1331	1773	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1332	1774
..	..	@@ -1337,7 +1779,7 @@
1337	1779	max_data->critical_start = data->critical_start;
1338	1780	max_data->critical_end = data->critical_end;
1339	1781
1340		- memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
	1782	+ strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1341	1783	max_data->pid = tsk->pid;
1342	1784	/*
1343	1785	* If tsk == current, then use current_uid(), as that does not use
..	..	@@ -1354,6 +1796,7 @@
1354	1796
1355	1797	/* record this tasks comm */
1356	1798	tracing_record_cmdline(tsk);
	1799	+ latency_fsnotify(tr);
1357	1800	}
1358	1801
1359	1802	/**
..	..	@@ -1361,12 +1804,14 @@
1361	1804	* @tr: tracer
1362	1805	* @tsk: the task with the latency
1363	1806	* @cpu: The cpu that initiated the trace.
	1807	+ * @cond_data: User data associated with a conditional snapshot
1364	1808	*
1365	1809	* Flip the buffers between the @tr and the max_tr and record information
1366	1810	* about which task was the cause of this latency.
1367	1811	*/
1368	1812	void
1369		-update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu)
	1813	+update_max_tr(struct trace_array tr, struct task_struct tsk, int cpu,
	1814	+ void *cond_data)
1370	1815	{
1371	1816	if (tr->stop_count)
1372	1817	return;
..	..	@@ -1381,23 +1826,29 @@
1381	1826
1382	1827	arch_spin_lock(&tr->max_lock);
1383	1828
1384		- /* Inherit the recordable setting from trace_buffer */
1385		- if (ring_buffer_record_is_set_on(tr->trace_buffer.buffer))
	1829	+ /* Inherit the recordable setting from array_buffer */
	1830	+ if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
1386	1831	ring_buffer_record_on(tr->max_buffer.buffer);
1387	1832	else
1388	1833	ring_buffer_record_off(tr->max_buffer.buffer);
1389	1834
1390		- swap(tr->trace_buffer.buffer, tr->max_buffer.buffer);
	1835	+#ifdef CONFIG_TRACER_SNAPSHOT
	1836	+ if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
	1837	+ goto out_unlock;
	1838	+#endif
	1839	+ swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1391	1840
1392	1841	__update_max_tr(tr, tsk, cpu);
	1842	+
	1843	+ out_unlock:
1393	1844	arch_spin_unlock(&tr->max_lock);
1394	1845	}
1395	1846
1396	1847	/**
1397	1848	* update_max_tr_single - only copy one trace over, and reset the rest
1398		- * @tr - tracer
1399		- * @tsk - task with the latency
1400		- * @cpu - the cpu of the buffer to copy.
	1849	+ * @tr: tracer
	1850	+ * @tsk: task with the latency
	1851	+ * @cpu: the cpu of the buffer to copy.
1401	1852	*
1402	1853	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
1403	1854	*/
..	..	@@ -1418,7 +1869,7 @@
1418	1869
1419	1870	arch_spin_lock(&tr->max_lock);
1420	1871
1421		- ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
	1872	+ ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
1422	1873
1423	1874	if (ret == -EBUSY) {
1424	1875	/*
..	..	@@ -1438,13 +1889,13 @@
1438	1889	}
1439	1890	#endif /* CONFIG_TRACER_MAX_TRACE */
1440	1891
1441		-static int wait_on_pipe(struct trace_iterator *iter, bool full)
	1892	+static int wait_on_pipe(struct trace_iterator *iter, int full)
1442	1893	{
1443	1894	/* Iterators are static, they should be filled or empty */
1444	1895	if (trace_buffer_iter(iter, iter->cpu_file))
1445	1896	return 0;
1446	1897
1447		- return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
	1898	+ return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file,
1448	1899	full);
1449	1900	}
1450	1901
..	..	@@ -1495,7 +1946,7 @@
1495	1946	* internal tracing to verify that everything is in order.
1496	1947	* If we fail, we do not register this tracer.
1497	1948	*/
1498		- tracing_reset_online_cpus(&tr->trace_buffer);
	1949	+ tracing_reset_online_cpus(&tr->array_buffer);
1499	1950
1500	1951	tr->current_trace = type;
1501	1952
..	..	@@ -1521,7 +1972,7 @@
1521	1972	return -1;
1522	1973	}
1523	1974	/* Only reset on passing, to avoid touching corrupted buffers */
1524		- tracing_reset_online_cpus(&tr->trace_buffer);
	1975	+ tracing_reset_online_cpus(&tr->array_buffer);
1525	1976
1526	1977	#ifdef CONFIG_TRACER_MAX_TRACE
1527	1978	if (type->use_max_tr) {
..	..	@@ -1555,6 +2006,10 @@
1555	2006
1556	2007	tracing_selftest_running = true;
1557	2008	list_for_each_entry_safe(p, n, &postponed_selftests, list) {
	2009	+ /* This loop can take minutes when sanitizers are enabled, so
	2010	+ * lets make sure we allow RCU processing.
	2011	+ */
	2012	+ cond_resched();
1558	2013	ret = run_tracer_selftest(p->type);
1559	2014	/* If the test fails, then warn and remove from available_tracers */
1560	2015	if (ret < 0) {
..	..	@@ -1593,7 +2048,7 @@
1593	2048
1594	2049	/**
1595	2050	* register_tracer - register a tracer with the ftrace system.
1596		- * @type - the plugin for the tracer
	2051	+ * @type: the plugin for the tracer
1597	2052	*
1598	2053	* Register a new plugin tracer.
1599	2054	*/
..	..	@@ -1610,6 +2065,12 @@
1610	2065	if (strlen(type->name) >= MAX_TRACER_SIZE) {
1611	2066	pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1612	2067	return -1;
	2068	+ }
	2069	+
	2070	+ if (security_locked_down(LOCKDOWN_TRACEFS)) {
	2071	+ pr_warn("Can not register tracer %s due to lockdown\n",
	2072	+ type->name);
	2073	+ return -EPERM;
1613	2074	}
1614	2075
1615	2076	mutex_lock(&trace_types_lock);
..	..	@@ -1670,19 +2131,15 @@
1670	2131	apply_trace_boot_options();
1671	2132
1672	2133	/* disable other selftests, since this will break it. */
1673		- tracing_selftest_disabled = true;
1674		-#ifdef CONFIG_FTRACE_STARTUP_TEST
1675		- printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1676		- type->name);
1677		-#endif
	2134	+ disable_tracing_selftest("running a tracer");
1678	2135
1679	2136	out_unlock:
1680	2137	return ret;
1681	2138	}
1682	2139
1683		-void tracing_reset(struct trace_buffer *buf, int cpu)
	2140	+static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
1684	2141	{
1685		- struct ring_buffer *buffer = buf->buffer;
	2142	+ struct trace_buffer *buffer = buf->buffer;
1686	2143
1687	2144	if (!buffer)
1688	2145	return;
..	..	@@ -1690,16 +2147,15 @@
1690	2147	ring_buffer_record_disable(buffer);
1691	2148
1692	2149	/* Make sure all commits have finished */
1693		- synchronize_sched();
	2150	+ synchronize_rcu();
1694	2151	ring_buffer_reset_cpu(buffer, cpu);
1695	2152
1696	2153	ring_buffer_record_enable(buffer);
1697	2154	}
1698	2155
1699		-void tracing_reset_online_cpus(struct trace_buffer *buf)
	2156	+void tracing_reset_online_cpus(struct array_buffer *buf)
1700	2157	{
1701		- struct ring_buffer *buffer = buf->buffer;
1702		- int cpu;
	2158	+ struct trace_buffer *buffer = buf->buffer;
1703	2159
1704	2160	if (!buffer)
1705	2161	return;
..	..	@@ -1707,12 +2163,11 @@
1707	2163	ring_buffer_record_disable(buffer);
1708	2164
1709	2165	/* Make sure all commits have finished */
1710		- synchronize_sched();
	2166	+ synchronize_rcu();
1711	2167
1712	2168	buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1713	2169
1714		- for_each_online_cpu(cpu)
1715		- ring_buffer_reset_cpu(buffer, cpu);
	2170	+ ring_buffer_reset_online_cpus(buffer);
1716	2171
1717	2172	ring_buffer_record_enable(buffer);
1718	2173	}
..	..	@@ -1726,7 +2181,7 @@
1726	2181	if (!tr->clear_trace)
1727	2182	continue;
1728	2183	tr->clear_trace = false;
1729		- tracing_reset_online_cpus(&tr->trace_buffer);
	2184	+ tracing_reset_online_cpus(&tr->array_buffer);
1730	2185	#ifdef CONFIG_TRACER_MAX_TRACE
1731	2186	tracing_reset_online_cpus(&tr->max_buffer);
1732	2187	#endif
..	..	@@ -1744,6 +2199,11 @@
1744	2199
1745	2200	#define SAVED_CMDLINES_DEFAULT 128
1746	2201	#define NO_CMDLINE_MAP UINT_MAX
	2202	+/*
	2203	+ * Preemption must be disabled before acquiring trace_cmdline_lock.
	2204	+ * The various trace_arrays' max_lock must be acquired in a context
	2205	+ * where interrupt is disabled.
	2206	+ */
1747	2207	static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1748	2208	struct saved_cmdlines_buffer {
1749	2209	unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
..	..	@@ -1761,7 +2221,7 @@
1761	2221
1762	2222	static inline void set_cmdline(int idx, const char *cmdline)
1763	2223	{
1764		- memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
	2224	+ strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1765	2225	}
1766	2226
1767	2227	static int allocate_cmdlines_buffer(unsigned int val,
..	..	@@ -1820,7 +2280,7 @@
1820	2280	*/
1821	2281	void tracing_start(void)
1822	2282	{
1823		- struct ring_buffer *buffer;
	2283	+ struct trace_buffer *buffer;
1824	2284	unsigned long flags;
1825	2285
1826	2286	if (tracing_disabled)
..	..	@@ -1839,7 +2299,7 @@
1839	2299	/* Prevent the buffers from switching */
1840	2300	arch_spin_lock(&global_trace.max_lock);
1841	2301
1842		- buffer = global_trace.trace_buffer.buffer;
	2302	+ buffer = global_trace.array_buffer.buffer;
1843	2303	if (buffer)
1844	2304	ring_buffer_record_enable(buffer);
1845	2305
..	..	@@ -1857,7 +2317,7 @@
1857	2317
1858	2318	static void tracing_start_tr(struct trace_array *tr)
1859	2319	{
1860		- struct ring_buffer *buffer;
	2320	+ struct trace_buffer *buffer;
1861	2321	unsigned long flags;
1862	2322
1863	2323	if (tracing_disabled)
..	..	@@ -1878,7 +2338,7 @@
1878	2338	goto out;
1879	2339	}
1880	2340
1881		- buffer = tr->trace_buffer.buffer;
	2341	+ buffer = tr->array_buffer.buffer;
1882	2342	if (buffer)
1883	2343	ring_buffer_record_enable(buffer);
1884	2344
..	..	@@ -1894,7 +2354,7 @@
1894	2354	*/
1895	2355	void tracing_stop(void)
1896	2356	{
1897		- struct ring_buffer *buffer;
	2357	+ struct trace_buffer *buffer;
1898	2358	unsigned long flags;
1899	2359
1900	2360	raw_spin_lock_irqsave(&global_trace.start_lock, flags);
..	..	@@ -1904,7 +2364,7 @@
1904	2364	/* Prevent the buffers from switching */
1905	2365	arch_spin_lock(&global_trace.max_lock);
1906	2366
1907		- buffer = global_trace.trace_buffer.buffer;
	2367	+ buffer = global_trace.array_buffer.buffer;
1908	2368	if (buffer)
1909	2369	ring_buffer_record_disable(buffer);
1910	2370
..	..	@@ -1922,7 +2382,7 @@
1922	2382
1923	2383	static void tracing_stop_tr(struct trace_array *tr)
1924	2384	{
1925		- struct ring_buffer *buffer;
	2385	+ struct trace_buffer *buffer;
1926	2386	unsigned long flags;
1927	2387
1928	2388	/* If global, we need to also stop the max tracer */
..	..	@@ -1933,7 +2393,7 @@
1933	2393	if (tr->stop_count++)
1934	2394	goto out;
1935	2395
1936		- buffer = tr->trace_buffer.buffer;
	2396	+ buffer = tr->array_buffer.buffer;
1937	2397	if (buffer)
1938	2398	ring_buffer_record_disable(buffer);
1939	2399
..	..	@@ -1956,7 +2416,11 @@
1956	2416	* the lock, but we also don't want to spin
1957	2417	* nor do we want to disable interrupts,
1958	2418	* so if we miss here, then better luck next time.
	2419	+ *
	2420	+ * This is called within the scheduler and wake up, so interrupts
	2421	+ * had better been disabled and run queue lock been held.
1959	2422	*/
	2423	+ lockdep_assert_preemption_disabled();
1960	2424	if (!arch_spin_trylock(&trace_cmdline_lock))
1961	2425	return 0;
1962	2426
..	..	@@ -2064,9 +2528,9 @@
2064	2528	/**
2065	2529	* tracing_record_taskinfo - record the task info of a task
2066	2530	*
2067		- * @task - task to record
2068		- * @flags - TRACE_RECORD_CMDLINE for recording comm
2069		- * - TRACE_RECORD_TGID for recording tgid
	2531	+ * @task: task to record
	2532	+ * @flags: TRACE_RECORD_CMDLINE for recording comm
	2533	+ * TRACE_RECORD_TGID for recording tgid
2070	2534	*/
2071	2535	void tracing_record_taskinfo(struct task_struct *task, int flags)
2072	2536	{
..	..	@@ -2092,10 +2556,10 @@
2092	2556	/**
2093	2557	* tracing_record_taskinfo_sched_switch - record task info for sched_switch
2094	2558	*
2095		- * @prev - previous task during sched_switch
2096		- * @next - next task during sched_switch
2097		- * @flags - TRACE_RECORD_CMDLINE for recording comm
2098		- * TRACE_RECORD_TGID for recording tgid
	2559	+ * @prev: previous task during sched_switch
	2560	+ * @next: next task during sched_switch
	2561	+ * @flags: TRACE_RECORD_CMDLINE for recording comm
	2562	+ * TRACE_RECORD_TGID for recording tgid
2099	2563	*/
2100	2564	void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2101	2565	struct task_struct *next, int flags)
..	..	@@ -2144,39 +2608,52 @@
2144	2608	}
2145	2609	EXPORT_SYMBOL_GPL(trace_handle_return);
2146	2610
2147		-void
2148		-tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
2149		- int pc)
	2611	+static unsigned short migration_disable_value(void)
2150	2612	{
2151		- struct task_struct *tsk = current;
2152		-
2153		- entry->preempt_count = pc & 0xff;
2154		- entry->preempt_lazy_count = preempt_lazy_count();
2155		- entry->pid = (tsk) ? tsk->pid : 0;
2156		- entry->flags =
2157		-#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2158		- (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) \|
	2613	+#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT_RT)
	2614	+ return current->migration_disabled;
2159	2615	#else
2160		- TRACE_FLAG_IRQS_NOSUPPORT \|
	2616	+ return 0;
2161	2617	#endif
2162		- ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) \|
2163		- ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) \|
2164		- ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) \|
2165		- (tif_need_resched_now() ? TRACE_FLAG_NEED_RESCHED : 0) \|
2166		- (need_resched_lazy() ? TRACE_FLAG_NEED_RESCHED_LAZY : 0) \|
2167		- (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
2168		-
2169		- entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0;
2170	2618	}
2171		-EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
	2619	+
	2620	+unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
	2621	+{
	2622	+ unsigned int trace_flags = irqs_status;
	2623	+ unsigned int pc;
	2624	+
	2625	+ pc = preempt_count();
	2626	+
	2627	+ if (pc & NMI_MASK)
	2628	+ trace_flags \|= TRACE_FLAG_NMI;
	2629	+ if (pc & HARDIRQ_MASK)
	2630	+ trace_flags \|= TRACE_FLAG_HARDIRQ;
	2631	+ if (in_serving_softirq())
	2632	+ trace_flags \|= TRACE_FLAG_SOFTIRQ;
	2633	+
	2634	+ if (tif_need_resched())
	2635	+ trace_flags \|= TRACE_FLAG_NEED_RESCHED;
	2636	+ if (test_preempt_need_resched())
	2637	+ trace_flags \|= TRACE_FLAG_PREEMPT_RESCHED;
	2638	+
	2639	+#ifdef CONFIG_PREEMPT_LAZY
	2640	+ if (need_resched_lazy())
	2641	+ trace_flags \|= TRACE_FLAG_NEED_RESCHED_LAZY;
	2642	+#endif
	2643	+
	2644	+ return (pc & 0xff) \|
	2645	+ (migration_disable_value() & 0xff) << 8 \|
	2646	+ (preempt_lazy_count() & 0xff) << 16 \|
	2647	+ (trace_flags << 24);
	2648	+}
2172	2649
2173	2650	struct ring_buffer_event *
2174		-trace_buffer_lock_reserve(struct ring_buffer *buffer,
	2651	+trace_buffer_lock_reserve(struct trace_buffer *buffer,
2175	2652	int type,
2176	2653	unsigned long len,
2177		- unsigned long flags, int pc)
	2654	+ unsigned int trace_ctx)
2178	2655	{
2179		- return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
	2656	+ return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx);
2180	2657	}
2181	2658
2182	2659	DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
..	..	@@ -2221,7 +2698,7 @@
2221	2698
2222	2699	preempt_disable();
2223	2700	if (cpu == smp_processor_id() &&
2224		- this_cpu_read(trace_buffered_event) !=
	2701	+ __this_cpu_read(trace_buffered_event) !=
2225	2702	per_cpu(trace_buffered_event, cpu))
2226	2703	WARN_ON_ONCE(1);
2227	2704	preempt_enable();
..	..	@@ -2271,7 +2748,7 @@
2271	2748	preempt_enable();
2272	2749
2273	2750	/* Wait for all current users to finish */
2274		- synchronize_sched();
	2751	+ synchronize_rcu();
2275	2752
2276	2753	for_each_tracing_cpu(cpu) {
2277	2754	free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
..	..	@@ -2290,18 +2767,18 @@
2290	2767	preempt_enable();
2291	2768	}
2292	2769
2293		-static struct ring_buffer *temp_buffer;
	2770	+static struct trace_buffer *temp_buffer;
2294	2771
2295	2772	struct ring_buffer_event *
2296		-trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
	2773	+trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2297	2774	struct trace_event_file *trace_file,
2298	2775	int type, unsigned long len,
2299		- unsigned long flags, int pc)
	2776	+ unsigned int trace_ctx)
2300	2777	{
2301	2778	struct ring_buffer_event *entry;
2302	2779	int val;
2303	2780
2304		- *current_rb = trace_file->tr->trace_buffer.buffer;
	2781	+ *current_rb = trace_file->tr->array_buffer.buffer;
2305	2782
2306	2783	if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
2307	2784	(EVENT_FILE_FL_SOFT_DISABLED \| EVENT_FILE_FL_FILTERED)) &&
..	..	@@ -2309,7 +2786,7 @@
2309	2786	/* Try to use the per cpu buffer first */
2310	2787	val = this_cpu_inc_return(trace_buffered_event_cnt);
2311	2788	if ((len < (PAGE_SIZE - sizeof(*entry) - sizeof(entry->array[0]))) && val == 1) {
2312		- trace_event_setup(entry, type, flags, pc);
	2789	+ trace_event_setup(entry, type, trace_ctx);
2313	2790	entry->array[0] = len;
2314	2791	return entry;
2315	2792	}
..	..	@@ -2317,28 +2794,29 @@
2317	2794	}
2318	2795
2319	2796	entry = __trace_buffer_lock_reserve(*current_rb,
2320		- type, len, flags, pc);
	2797	+ type, len, trace_ctx);
2321	2798	/*
2322	2799	* If tracing is off, but we have triggers enabled
2323	2800	* we still need to look at the event data. Use the temp_buffer
2324		- * to store the trace event for the tigger to use. It's recusive
	2801	+ * to store the trace event for the trigger to use. It's recursive
2325	2802	* safe and will not be recorded anywhere.
2326	2803	*/
2327	2804	if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2328	2805	*current_rb = temp_buffer;
2329		- entry = __trace_buffer_lock_reserve(*current_rb,
2330		- type, len, flags, pc);
	2806	+ entry = __trace_buffer_lock_reserve(*current_rb, type, len,
	2807	+ trace_ctx);
2331	2808	}
2332	2809	return entry;
2333	2810	}
2334	2811	EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2335	2812
2336		-static DEFINE_SPINLOCK(tracepoint_iter_lock);
	2813	+static DEFINE_RAW_SPINLOCK(tracepoint_iter_lock);
2337	2814	static DEFINE_MUTEX(tracepoint_printk_mutex);
2338	2815
2339	2816	static void output_printk(struct trace_event_buffer *fbuffer)
2340	2817	{
2341	2818	struct trace_event_call *event_call;
	2819	+ struct trace_event_file *file;
2342	2820	struct trace_event *event;
2343	2821	unsigned long flags;
2344	2822	struct trace_iterator *iter = tracepoint_print_iter;
..	..	@@ -2352,20 +2830,26 @@
2352	2830	!event_call->event.funcs->trace)
2353	2831	return;
2354	2832
	2833	+ file = fbuffer->trace_file;
	2834	+ if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) \|\|
	2835	+ (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
	2836	+ !filter_match_preds(file->filter, fbuffer->entry)))
	2837	+ return;
	2838	+
2355	2839	event = &fbuffer->trace_file->event_call->event;
2356	2840
2357		- spin_lock_irqsave(&tracepoint_iter_lock, flags);
	2841	+ raw_spin_lock_irqsave(&tracepoint_iter_lock, flags);
2358	2842	trace_seq_init(&iter->seq);
2359	2843	iter->ent = fbuffer->entry;
2360	2844	event_call->event.funcs->trace(iter, 0, event);
2361	2845	trace_seq_putc(&iter->seq, 0);
2362	2846	printk("%s", iter->seq.buffer);
2363	2847
2364		- spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
	2848	+ raw_spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2365	2849	}
2366	2850
2367	2851	int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2368		- void __user buffer, size_t lenp,
	2852	+ void buffer, size_t lenp,
2369	2853	loff_t *ppos)
2370	2854	{
2371	2855	int save_tracepoint_printk;
..	..	@@ -2402,9 +2886,11 @@
2402	2886	if (static_key_false(&tracepoint_printk_key.key))
2403	2887	output_printk(fbuffer);
2404	2888
2405		- event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
	2889	+ if (static_branch_unlikely(&trace_event_exports_enabled))
	2890	+ ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);
	2891	+ event_trigger_unlock_commit_regs(fbuffer->trace_file, fbuffer->buffer,
2406	2892	fbuffer->event, fbuffer->entry,
2407		- fbuffer->flags, fbuffer->pc);
	2893	+ fbuffer->trace_ctx, fbuffer->regs);
2408	2894	}
2409	2895	EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2410	2896
..	..	@@ -2418,9 +2904,9 @@
2418	2904	# define STACK_SKIP 3
2419	2905
2420	2906	void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2421		- struct ring_buffer *buffer,
	2907	+ struct trace_buffer *buffer,
2422	2908	struct ring_buffer_event *event,
2423		- unsigned long flags, int pc,
	2909	+ unsigned int trace_ctx,
2424	2910	struct pt_regs *regs)
2425	2911	{
2426	2912	__buffer_unlock_commit(buffer, event);
..	..	@@ -2431,155 +2917,31 @@
2431	2917	* and mmiotrace, but that's ok if they lose a function or
2432	2918	* two. They are not that meaningful.
2433	2919	*/
2434		- ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs);
2435		- ftrace_trace_userstack(tr, buffer, flags, pc);
	2920	+ ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs);
	2921	+ ftrace_trace_userstack(tr, buffer, trace_ctx);
2436	2922	}
2437	2923
2438	2924	/*
2439	2925	* Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2440	2926	*/
2441	2927	void
2442		-trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
	2928	+trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2443	2929	struct ring_buffer_event *event)
2444	2930	{
2445	2931	__buffer_unlock_commit(buffer, event);
2446	2932	}
2447	2933
2448		-static void
2449		-trace_process_export(struct trace_export *export,
2450		- struct ring_buffer_event *event)
2451		-{
2452		- struct trace_entry *entry;
2453		- unsigned int size = 0;
2454		-
2455		- entry = ring_buffer_event_data(event);
2456		- size = ring_buffer_event_length(event);
2457		- export->write(export, entry, size);
2458		-}
2459		-
2460		-static DEFINE_MUTEX(ftrace_export_lock);
2461		-
2462		-static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2463		-
2464		-static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2465		-
2466		-static inline void ftrace_exports_enable(void)
2467		-{
2468		- static_branch_enable(&ftrace_exports_enabled);
2469		-}
2470		-
2471		-static inline void ftrace_exports_disable(void)
2472		-{
2473		- static_branch_disable(&ftrace_exports_enabled);
2474		-}
2475		-
2476		-void ftrace_exports(struct ring_buffer_event *event)
2477		-{
2478		- struct trace_export *export;
2479		-
2480		- preempt_disable_notrace();
2481		-
2482		- export = rcu_dereference_raw_notrace(ftrace_exports_list);
2483		- while (export) {
2484		- trace_process_export(export, event);
2485		- export = rcu_dereference_raw_notrace(export->next);
2486		- }
2487		-
2488		- preempt_enable_notrace();
2489		-}
2490		-
2491		-static inline void
2492		-add_trace_export(struct trace_export *list, struct trace_export export)
2493		-{
2494		- rcu_assign_pointer(export->next, *list);
2495		- /*
2496		- * We are entering export into the list but another
2497		- * CPU might be walking that list. We need to make sure
2498		- * the export->next pointer is valid before another CPU sees
2499		- * the export pointer included into the list.
2500		- */
2501		- rcu_assign_pointer(*list, export);
2502		-}
2503		-
2504		-static inline int
2505		-rm_trace_export(struct trace_export *list, struct trace_export export)
2506		-{
2507		- struct trace_export **p;
2508		-
2509		- for (p = list; p != NULL; p = &(p)->next)
2510		- if (*p == export)
2511		- break;
2512		-
2513		- if (*p != export)
2514		- return -1;
2515		-
2516		- rcu_assign_pointer(p, (p)->next);
2517		-
2518		- return 0;
2519		-}
2520		-
2521		-static inline void
2522		-add_ftrace_export(struct trace_export *list, struct trace_export export)
2523		-{
2524		- if (*list == NULL)
2525		- ftrace_exports_enable();
2526		-
2527		- add_trace_export(list, export);
2528		-}
2529		-
2530		-static inline int
2531		-rm_ftrace_export(struct trace_export *list, struct trace_export export)
2532		-{
2533		- int ret;
2534		-
2535		- ret = rm_trace_export(list, export);
2536		- if (*list == NULL)
2537		- ftrace_exports_disable();
2538		-
2539		- return ret;
2540		-}
2541		-
2542		-int register_ftrace_export(struct trace_export *export)
2543		-{
2544		- if (WARN_ON_ONCE(!export->write))
2545		- return -1;
2546		-
2547		- mutex_lock(&ftrace_export_lock);
2548		-
2549		- add_ftrace_export(&ftrace_exports_list, export);
2550		-
2551		- mutex_unlock(&ftrace_export_lock);
2552		-
2553		- return 0;
2554		-}
2555		-EXPORT_SYMBOL_GPL(register_ftrace_export);
2556		-
2557		-int unregister_ftrace_export(struct trace_export *export)
2558		-{
2559		- int ret;
2560		-
2561		- mutex_lock(&ftrace_export_lock);
2562		-
2563		- ret = rm_ftrace_export(&ftrace_exports_list, export);
2564		-
2565		- mutex_unlock(&ftrace_export_lock);
2566		-
2567		- return ret;
2568		-}
2569		-EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2570		-
2571	2934	void
2572		-trace_function(struct trace_array *tr,
2573		- unsigned long ip, unsigned long parent_ip, unsigned long flags,
2574		- int pc)
	2935	+trace_function(struct trace_array *tr, unsigned long ip, unsigned long
	2936	+ parent_ip, unsigned int trace_ctx)
2575	2937	{
2576	2938	struct trace_event_call *call = &event_function;
2577		- struct ring_buffer *buffer = tr->trace_buffer.buffer;
	2939	+ struct trace_buffer *buffer = tr->array_buffer.buffer;
2578	2940	struct ring_buffer_event *event;
2579	2941	struct ftrace_entry *entry;
2580	2942
2581	2943	event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2582		- flags, pc);
	2944	+ trace_ctx);
2583	2945	if (!event)
2584	2946	return;
2585	2947	entry = ring_buffer_event_data(event);
..	..	@@ -2587,35 +2949,41 @@
2587	2949	entry->parent_ip = parent_ip;
2588	2950
2589	2951	if (!call_filter_check_discard(call, entry, buffer, event)) {
2590		- if (static_branch_unlikely(&ftrace_exports_enabled))
2591		- ftrace_exports(event);
	2952	+ if (static_branch_unlikely(&trace_function_exports_enabled))
	2953	+ ftrace_exports(event, TRACE_EXPORT_FUNCTION);
2592	2954	__buffer_unlock_commit(buffer, event);
2593	2955	}
2594	2956	}
2595	2957
2596	2958	#ifdef CONFIG_STACKTRACE
2597	2959
2598		-#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
	2960	+/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
	2961	+#define FTRACE_KSTACK_NESTING 4
	2962	+
	2963	+#define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
	2964	+
2599	2965	struct ftrace_stack {
2600		- unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
	2966	+ unsigned long calls[FTRACE_KSTACK_ENTRIES];
2601	2967	};
2602	2968
2603		-static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
	2969	+
	2970	+struct ftrace_stacks {
	2971	+ struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
	2972	+};
	2973	+
	2974	+static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2604	2975	static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2605	2976
2606		-static void __ftrace_trace_stack(struct ring_buffer *buffer,
2607		- unsigned long flags,
2608		- int skip, int pc, struct pt_regs *regs)
	2977	+static void __ftrace_trace_stack(struct trace_buffer *buffer,
	2978	+ unsigned int trace_ctx,
	2979	+ int skip, struct pt_regs *regs)
2609	2980	{
2610	2981	struct trace_event_call *call = &event_kernel_stack;
2611	2982	struct ring_buffer_event *event;
	2983	+ unsigned int size, nr_entries;
	2984	+ struct ftrace_stack *fstack;
2612	2985	struct stack_entry *entry;
2613		- struct stack_trace trace;
2614		- int use_stack;
2615		- int size = FTRACE_STACK_ENTRIES;
2616		-
2617		- trace.nr_entries = 0;
2618		- trace.skip = skip;
	2986	+ int stackidx;
2619	2987
2620	2988	/*
2621	2989	* Add one, for this function and the call to save_stack_trace()
..	..	@@ -2623,65 +2991,46 @@
2623	2991	*/
2624	2992	#ifndef CONFIG_UNWINDER_ORC
2625	2993	if (!regs)
2626		- trace.skip++;
	2994	+ skip++;
2627	2995	#endif
2628	2996
2629		- /*
2630		- * Since events can happen in NMIs there's no safe way to
2631		- * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2632		- * or NMI comes in, it will just have to use the default
2633		- * FTRACE_STACK_SIZE.
2634		- */
2635	2997	preempt_disable_notrace();
2636	2998
2637		- use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
	2999	+ stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
	3000	+
	3001	+ /* This should never happen. If it does, yell once and skip */
	3002	+ if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
	3003	+ goto out;
	3004	+
2638	3005	/*
2639		- * We don't need any atomic variables, just a barrier.
2640		- * If an interrupt comes in, we don't care, because it would
2641		- * have exited and put the counter back to what we want.
2642		- * We just need a barrier to keep gcc from moving things
2643		- * around.
	3006	+ * The above __this_cpu_inc_return() is 'atomic' cpu local. An
	3007	+ * interrupt will either see the value pre increment or post
	3008	+ * increment. If the interrupt happens pre increment it will have
	3009	+ * restored the counter when it returns. We just need a barrier to
	3010	+ * keep gcc from moving things around.
2644	3011	*/
2645	3012	barrier();
2646		- if (use_stack == 1) {
2647		- trace.entries = this_cpu_ptr(ftrace_stack.calls);
2648		- trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
2649	3013
2650		- if (regs)
2651		- save_stack_trace_regs(regs, &trace);
2652		- else
2653		- save_stack_trace(&trace);
	3014	+ fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
	3015	+ size = ARRAY_SIZE(fstack->calls);
2654	3016
2655		- if (trace.nr_entries > size)
2656		- size = trace.nr_entries;
2657		- } else
2658		- /* From now on, use_stack is a boolean */
2659		- use_stack = 0;
	3017	+ if (regs) {
	3018	+ nr_entries = stack_trace_save_regs(regs, fstack->calls,
	3019	+ size, skip);
	3020	+ } else {
	3021	+ nr_entries = stack_trace_save(fstack->calls, size, skip);
	3022	+ }
2660	3023
2661		- size *= sizeof(unsigned long);
2662		-
	3024	+ size = nr_entries * sizeof(unsigned long);
2663	3025	event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2664	3026	(sizeof(*entry) - sizeof(entry->caller)) + size,
2665		- flags, pc);
	3027	+ trace_ctx);
2666	3028	if (!event)
2667	3029	goto out;
2668	3030	entry = ring_buffer_event_data(event);
2669	3031
2670		- memset(&entry->caller, 0, size);
2671		-
2672		- if (use_stack)
2673		- memcpy(&entry->caller, trace.entries,
2674		- trace.nr_entries * sizeof(unsigned long));
2675		- else {
2676		- trace.max_entries = FTRACE_STACK_ENTRIES;
2677		- trace.entries = entry->caller;
2678		- if (regs)
2679		- save_stack_trace_regs(regs, &trace);
2680		- else
2681		- save_stack_trace(&trace);
2682		- }
2683		-
2684		- entry->size = trace.nr_entries;
	3032	+ memcpy(&entry->caller, fstack->calls, size);
	3033	+ entry->size = nr_entries;
2685	3034
2686	3035	if (!call_filter_check_discard(call, entry, buffer, event))
2687	3036	__buffer_unlock_commit(buffer, event);
..	..	@@ -2695,23 +3044,23 @@
2695	3044	}
2696	3045
2697	3046	static inline void ftrace_trace_stack(struct trace_array *tr,
2698		- struct ring_buffer *buffer,
2699		- unsigned long flags,
2700		- int skip, int pc, struct pt_regs *regs)
	3047	+ struct trace_buffer *buffer,
	3048	+ unsigned int trace_ctx,
	3049	+ int skip, struct pt_regs *regs)
2701	3050	{
2702	3051	if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2703	3052	return;
2704	3053
2705		- __ftrace_trace_stack(buffer, flags, skip, pc, regs);
	3054	+ __ftrace_trace_stack(buffer, trace_ctx, skip, regs);
2706	3055	}
2707	3056
2708		-void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2709		- int pc)
	3057	+void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
	3058	+ int skip)
2710	3059	{
2711		- struct ring_buffer *buffer = tr->trace_buffer.buffer;
	3060	+ struct trace_buffer *buffer = tr->array_buffer.buffer;
2712	3061
2713	3062	if (rcu_is_watching()) {
2714		- __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
	3063	+ __ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
2715	3064	return;
2716	3065	}
2717	3066
..	..	@@ -2725,7 +3074,7 @@
2725	3074	return;
2726	3075
2727	3076	rcu_irq_enter_irqson();
2728		- __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
	3077	+ __ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
2729	3078	rcu_irq_exit_irqson();
2730	3079	}
2731	3080
..	..	@@ -2735,31 +3084,28 @@
2735	3084	*/
2736	3085	void trace_dump_stack(int skip)
2737	3086	{
2738		- unsigned long flags;
2739		-
2740	3087	if (tracing_disabled \|\| tracing_selftest_running)
2741	3088	return;
2742		-
2743		- local_save_flags(flags);
2744	3089
2745	3090	#ifndef CONFIG_UNWINDER_ORC
2746	3091	/* Skip 1 to skip this function. */
2747	3092	skip++;
2748	3093	#endif
2749		- __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2750		- flags, skip, preempt_count(), NULL);
	3094	+ __ftrace_trace_stack(global_trace.array_buffer.buffer,
	3095	+ tracing_gen_ctx(), skip, NULL);
2751	3096	}
	3097	+EXPORT_SYMBOL_GPL(trace_dump_stack);
2752	3098
	3099	+#ifdef CONFIG_USER_STACKTRACE_SUPPORT
2753	3100	static DEFINE_PER_CPU(int, user_stack_count);
2754	3101
2755		-void
	3102	+static void
2756	3103	ftrace_trace_userstack(struct trace_array *tr,
2757		- struct ring_buffer *buffer, unsigned long flags, int pc)
	3104	+ struct trace_buffer *buffer, unsigned int trace_ctx)
2758	3105	{
2759	3106	struct trace_event_call *call = &event_user_stack;
2760	3107	struct ring_buffer_event *event;
2761	3108	struct userstack_entry *entry;
2762		- struct stack_trace trace;
2763	3109
2764	3110	if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
2765	3111	return;
..	..	@@ -2782,7 +3128,7 @@
2782	3128	__this_cpu_inc(user_stack_count);
2783	3129
2784	3130	event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2785		- sizeof(*entry), flags, pc);
	3131	+ sizeof(*entry), trace_ctx);
2786	3132	if (!event)
2787	3133	goto out_drop_count;
2788	3134	entry = ring_buffer_event_data(event);
..	..	@@ -2790,12 +3136,7 @@
2790	3136	entry->tgid = current->tgid;
2791	3137	memset(&entry->caller, 0, sizeof(entry->caller));
2792	3138
2793		- trace.nr_entries = 0;
2794		- trace.max_entries = FTRACE_STACK_ENTRIES;
2795		- trace.skip = 0;
2796		- trace.entries = entry->caller;
2797		-
2798		- save_stack_trace_user(&trace);
	3139	+ stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
2799	3140	if (!call_filter_check_discard(call, entry, buffer, event))
2800	3141	__buffer_unlock_commit(buffer, event);
2801	3142
..	..	@@ -2804,13 +3145,13 @@
2804	3145	out:
2805	3146	preempt_enable();
2806	3147	}
2807		-
2808		-#ifdef UNUSED
2809		-static void __trace_userstack(struct trace_array *tr, unsigned long flags)
	3148	+#else /* CONFIG_USER_STACKTRACE_SUPPORT */
	3149	+static void ftrace_trace_userstack(struct trace_array *tr,
	3150	+ struct trace_buffer *buffer,
	3151	+ unsigned int trace_ctx)
2810	3152	{
2811		- ftrace_trace_userstack(tr, flags, preempt_count());
2812	3153	}
2813		-#endif /* UNUSED */
	3154	+#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
2814	3155
2815	3156	#endif /* CONFIG_STACKTRACE */
2816	3157
..	..	@@ -2851,8 +3192,11 @@
2851	3192	{
2852	3193	struct trace_buffer_struct __percpu *buffers;
2853	3194
	3195	+ if (trace_percpu_buffer)
	3196	+ return 0;
	3197	+
2854	3198	buffers = alloc_percpu(struct trace_buffer_struct);
2855		- if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
	3199	+ if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
2856	3200	return -ENOMEM;
2857	3201
2858	3202	trace_percpu_buffer = buffers;
..	..	@@ -2897,9 +3241,10 @@
2897	3241	* directly here. If the global_trace.buffer is already
2898	3242	* allocated here, then this was called by module code.
2899	3243	*/
2900		- if (global_trace.trace_buffer.buffer)
	3244	+ if (global_trace.array_buffer.buffer)
2901	3245	tracing_start_cmdline_record();
2902	3246	}
	3247	+EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
2903	3248
2904	3249	void trace_printk_start_comm(void)
2905	3250	{
..	..	@@ -2922,18 +3267,20 @@
2922	3267
2923	3268	/**
2924	3269	* trace_vbprintk - write binary msg to tracing buffer
2925		- *
	3270	+ * @ip: The address of the caller
	3271	+ * @fmt: The string format to write to the buffer
	3272	+ * @args: Arguments for @fmt
2926	3273	*/
2927	3274	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2928	3275	{
2929	3276	struct trace_event_call *call = &event_bprint;
2930	3277	struct ring_buffer_event *event;
2931		- struct ring_buffer *buffer;
	3278	+ struct trace_buffer *buffer;
2932	3279	struct trace_array *tr = &global_trace;
2933	3280	struct bprint_entry *entry;
2934		- unsigned long flags;
	3281	+ unsigned int trace_ctx;
2935	3282	char *tbuffer;
2936		- int len = 0, size, pc;
	3283	+ int len = 0, size;
2937	3284
2938	3285	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
2939	3286	return 0;
..	..	@@ -2941,7 +3288,7 @@
2941	3288	/* Don't pollute graph traces with trace_vprintk internals */
2942	3289	pause_graph_tracing();
2943	3290
2944		- pc = preempt_count();
	3291	+ trace_ctx = tracing_gen_ctx();
2945	3292	preempt_disable_notrace();
2946	3293
2947	3294	tbuffer = get_trace_buf();
..	..	@@ -2953,13 +3300,13 @@
2953	3300	len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2954	3301
2955	3302	if (len > TRACE_BUF_SIZE/sizeof(int) \|\| len < 0)
2956		- goto out;
	3303	+ goto out_put;
2957	3304
2958		- local_save_flags(flags);
2959	3305	size = sizeof(entry) + sizeof(u32) len;
2960		- buffer = tr->trace_buffer.buffer;
	3306	+ buffer = tr->array_buffer.buffer;
	3307	+ ring_buffer_nest_start(buffer);
2961	3308	event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2962		- flags, pc);
	3309	+ trace_ctx);
2963	3310	if (!event)
2964	3311	goto out;
2965	3312	entry = ring_buffer_event_data(event);
..	..	@@ -2969,10 +3316,12 @@
2969	3316	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2970	3317	if (!call_filter_check_discard(call, entry, buffer, event)) {
2971	3318	__buffer_unlock_commit(buffer, event);
2972		- ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
	3319	+ ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL);
2973	3320	}
2974	3321
2975	3322	out:
	3323	+ ring_buffer_nest_end(buffer);
	3324	+out_put:
2976	3325	put_trace_buf();
2977	3326
2978	3327	out_nobuffer:
..	..	@@ -2985,14 +3334,14 @@
2985	3334
2986	3335	__printf(3, 0)
2987	3336	static int
2988		-__trace_array_vprintk(struct ring_buffer *buffer,
	3337	+__trace_array_vprintk(struct trace_buffer *buffer,
2989	3338	unsigned long ip, const char *fmt, va_list args)
2990	3339	{
2991	3340	struct trace_event_call *call = &event_print;
2992	3341	struct ring_buffer_event *event;
2993		- int len = 0, size, pc;
	3342	+ int len = 0, size;
2994	3343	struct print_entry *entry;
2995		- unsigned long flags;
	3344	+ unsigned int trace_ctx;
2996	3345	char *tbuffer;
2997	3346
2998	3347	if (tracing_disabled \|\| tracing_selftest_running)
..	..	@@ -3001,7 +3350,7 @@
3001	3350	/* Don't pollute graph traces with trace_vprintk internals */
3002	3351	pause_graph_tracing();
3003	3352
3004		- pc = preempt_count();
	3353	+ trace_ctx = tracing_gen_ctx();
3005	3354	preempt_disable_notrace();
3006	3355
3007	3356
..	..	@@ -3013,10 +3362,10 @@
3013	3362
3014	3363	len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3015	3364
3016		- local_save_flags(flags);
3017	3365	size = sizeof(*entry) + len + 1;
	3366	+ ring_buffer_nest_start(buffer);
3018	3367	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3019		- flags, pc);
	3368	+ trace_ctx);
3020	3369	if (!event)
3021	3370	goto out;
3022	3371	entry = ring_buffer_event_data(event);
..	..	@@ -3025,10 +3374,11 @@
3025	3374	memcpy(&entry->buf, tbuffer, len + 1);
3026	3375	if (!call_filter_check_discard(call, entry, buffer, event)) {
3027	3376	__buffer_unlock_commit(buffer, event);
3028		- ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
	3377	+ ftrace_trace_stack(&global_trace, buffer, trace_ctx, 6, NULL);
3029	3378	}
3030	3379
3031	3380	out:
	3381	+ ring_buffer_nest_end(buffer);
3032	3382	put_trace_buf();
3033	3383
3034	3384	out_nobuffer:
..	..	@@ -3042,9 +3392,29 @@
3042	3392	int trace_array_vprintk(struct trace_array *tr,
3043	3393	unsigned long ip, const char *fmt, va_list args)
3044	3394	{
3045		- return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
	3395	+ return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
3046	3396	}
3047	3397
	3398	+/**
	3399	+ * trace_array_printk - Print a message to a specific instance
	3400	+ * @tr: The instance trace_array descriptor
	3401	+ * @ip: The instruction pointer that this is called from.
	3402	+ * @fmt: The format to print (printf format)
	3403	+ *
	3404	+ * If a subsystem sets up its own instance, they have the right to
	3405	+ * printk strings into their tracing instance buffer using this
	3406	+ * function. Note, this function will not write into the top level
	3407	+ * buffer (use trace_printk() for that), as writing into the top level
	3408	+ * buffer should only have events that can be individually disabled.
	3409	+ * trace_printk() is only used for debugging a kernel, and should not
	3410	+ * be ever encorporated in normal use.
	3411	+ *
	3412	+ * trace_array_printk() can be used, as it will not add noise to the
	3413	+ * top level tracing buffer.
	3414	+ *
	3415	+ * Note, trace_array_init_printk() must be called on @tr before this
	3416	+ * can be used.
	3417	+ */
3048	3418	__printf(3, 0)
3049	3419	int trace_array_printk(struct trace_array *tr,
3050	3420	unsigned long ip, const char *fmt, ...)
..	..	@@ -3052,20 +3422,46 @@
3052	3422	int ret;
3053	3423	va_list ap;
3054	3424
3055		- if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3056		- return 0;
3057		-
3058	3425	if (!tr)
3059	3426	return -ENOENT;
	3427	+
	3428	+ /* This is only allowed for created instances */
	3429	+ if (tr == &global_trace)
	3430	+ return 0;
	3431	+
	3432	+ if (!(tr->trace_flags & TRACE_ITER_PRINTK))
	3433	+ return 0;
3060	3434
3061	3435	va_start(ap, fmt);
3062	3436	ret = trace_array_vprintk(tr, ip, fmt, ap);
3063	3437	va_end(ap);
3064	3438	return ret;
3065	3439	}
	3440	+EXPORT_SYMBOL_GPL(trace_array_printk);
	3441	+
	3442	+/**
	3443	+ * trace_array_init_printk - Initialize buffers for trace_array_printk()
	3444	+ * @tr: The trace array to initialize the buffers for
	3445	+ *
	3446	+ * As trace_array_printk() only writes into instances, they are OK to
	3447	+ * have in the kernel (unlike trace_printk()). This needs to be called
	3448	+ * before trace_array_printk() can be used on a trace_array.
	3449	+ */
	3450	+int trace_array_init_printk(struct trace_array *tr)
	3451	+{
	3452	+ if (!tr)
	3453	+ return -ENOENT;
	3454	+
	3455	+ /* This is only allowed for created instances */
	3456	+ if (tr == &global_trace)
	3457	+ return -EINVAL;
	3458	+
	3459	+ return alloc_percpu_trace_buffer();
	3460	+}
	3461	+EXPORT_SYMBOL_GPL(trace_array_init_printk);
3066	3462
3067	3463	__printf(3, 4)
3068		-int trace_array_printk_buf(struct ring_buffer *buffer,
	3464	+int trace_array_printk_buf(struct trace_buffer *buffer,
3069	3465	unsigned long ip, const char *fmt, ...)
3070	3466	{
3071	3467	int ret;
..	..	@@ -3093,7 +3489,7 @@
3093	3489
3094	3490	iter->idx++;
3095	3491	if (buf_iter)
3096		- ring_buffer_read(buf_iter, NULL);
	3492	+ ring_buffer_iter_advance(buf_iter);
3097	3493	}
3098	3494
3099	3495	static struct trace_entry *
..	..	@@ -3103,11 +3499,15 @@
3103	3499	struct ring_buffer_event *event;
3104	3500	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3105	3501
3106		- if (buf_iter)
	3502	+ if (buf_iter) {
3107	3503	event = ring_buffer_iter_peek(buf_iter, ts);
3108		- else
3109		- event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
	3504	+ if (lost_events)
	3505	+ *lost_events = ring_buffer_iter_dropped(buf_iter) ?
	3506	+ (unsigned long)-1 : 0;
	3507	+ } else {
	3508	+ event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
3110	3509	lost_events);
	3510	+ }
3111	3511
3112	3512	if (event) {
3113	3513	iter->ent_size = ring_buffer_event_length(event);
..	..	@@ -3121,7 +3521,7 @@
3121	3521	__find_next_entry(struct trace_iterator iter, int ent_cpu,
3122	3522	unsigned long missing_events, u64 ent_ts)
3123	3523	{
3124		- struct ring_buffer *buffer = iter->trace_buffer->buffer;
	3524	+ struct trace_buffer *buffer = iter->array_buffer->buffer;
3125	3525	struct trace_entry ent, next = NULL;
3126	3526	unsigned long lost_events = 0, next_lost = 0;
3127	3527	int cpu_file = iter->cpu_file;
..	..	@@ -3177,11 +3577,53 @@
3177	3577	return next;
3178	3578	}
3179	3579
	3580	+#define STATIC_TEMP_BUF_SIZE 128
	3581	+static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
	3582	+
3180	3583	/* Find the next real entry, without updating the iterator itself */
3181	3584	struct trace_entry trace_find_next_entry(struct trace_iterator iter,
3182	3585	int ent_cpu, u64 ent_ts)
3183	3586	{
3184		- return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
	3587	+ /* __find_next_entry will reset ent_size */
	3588	+ int ent_size = iter->ent_size;
	3589	+ struct trace_entry *entry;
	3590	+
	3591	+ /*
	3592	+ * If called from ftrace_dump(), then the iter->temp buffer
	3593	+ * will be the static_temp_buf and not created from kmalloc.
	3594	+ * If the entry size is greater than the buffer, we can
	3595	+ * not save it. Just return NULL in that case. This is only
	3596	+ * used to add markers when two consecutive events' time
	3597	+ * stamps have a large delta. See trace_print_lat_context()
	3598	+ */
	3599	+ if (iter->temp == static_temp_buf &&
	3600	+ STATIC_TEMP_BUF_SIZE < ent_size)
	3601	+ return NULL;
	3602	+
	3603	+ /*
	3604	+ * The __find_next_entry() may call peek_next_entry(), which may
	3605	+ * call ring_buffer_peek() that may make the contents of iter->ent
	3606	+ * undefined. Need to copy iter->ent now.
	3607	+ */
	3608	+ if (iter->ent && iter->ent != iter->temp) {
	3609	+ if ((!iter->temp \|\| iter->temp_size < iter->ent_size) &&
	3610	+ !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
	3611	+ void *temp;
	3612	+ temp = kmalloc(iter->ent_size, GFP_KERNEL);
	3613	+ if (!temp)
	3614	+ return NULL;
	3615	+ kfree(iter->temp);
	3616	+ iter->temp = temp;
	3617	+ iter->temp_size = iter->ent_size;
	3618	+ }
	3619	+ memcpy(iter->temp, iter->ent, iter->ent_size);
	3620	+ iter->ent = iter->temp;
	3621	+ }
	3622	+ entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
	3623	+ /* Put back the original ent_size */
	3624	+ iter->ent_size = ent_size;
	3625	+
	3626	+ return entry;
3185	3627	}
3186	3628
3187	3629	/* Find the next real entry, and increment the iterator to the next entry */
..	..	@@ -3198,7 +3640,7 @@
3198	3640
3199	3641	static void trace_consume(struct trace_iterator *iter)
3200	3642	{
3201		- ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
	3643	+ ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
3202	3644	&iter->lost_events);
3203	3645	}
3204	3646
..	..	@@ -3231,12 +3673,11 @@
3231	3673
3232	3674	void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3233	3675	{
3234		- struct ring_buffer_event *event;
3235	3676	struct ring_buffer_iter *buf_iter;
3236	3677	unsigned long entries = 0;
3237	3678	u64 ts;
3238	3679
3239		- per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
	3680	+ per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
3240	3681
3241	3682	buf_iter = trace_buffer_iter(iter, cpu);
3242	3683	if (!buf_iter)
..	..	@@ -3249,14 +3690,14 @@
3249	3690	* that a reset never took place on a cpu. This is evident
3250	3691	* by the timestamp being before the start of the buffer.
3251	3692	*/
3252		- while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
3253		- if (ts >= iter->trace_buffer->time_start)
	3693	+ while (ring_buffer_iter_peek(buf_iter, &ts)) {
	3694	+ if (ts >= iter->array_buffer->time_start)
3254	3695	break;
3255	3696	entries++;
3256		- ring_buffer_read(buf_iter, NULL);
	3697	+ ring_buffer_iter_advance(buf_iter);
3257	3698	}
3258	3699
3259		- per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
	3700	+ per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
3260	3701	}
3261	3702
3262	3703	/*
..	..	@@ -3335,49 +3776,84 @@
3335	3776	}
3336	3777
3337	3778	static void
3338		-get_total_entries(struct trace_buffer *buf,
3339		- unsigned long total, unsigned long entries)
	3779	+get_total_entries_cpu(struct array_buffer buf, unsigned long total,
	3780	+ unsigned long *entries, int cpu)
3340	3781	{
3341	3782	unsigned long count;
	3783	+
	3784	+ count = ring_buffer_entries_cpu(buf->buffer, cpu);
	3785	+ /*
	3786	+ * If this buffer has skipped entries, then we hold all
	3787	+ * entries for the trace and we need to ignore the
	3788	+ * ones before the time stamp.
	3789	+ */
	3790	+ if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
	3791	+ count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
	3792	+ /* total is the same as the entries */
	3793	+ *total = count;
	3794	+ } else
	3795	+ *total = count +
	3796	+ ring_buffer_overrun_cpu(buf->buffer, cpu);
	3797	+ *entries = count;
	3798	+}
	3799	+
	3800	+static void
	3801	+get_total_entries(struct array_buffer *buf,
	3802	+ unsigned long total, unsigned long entries)
	3803	+{
	3804	+ unsigned long t, e;
3342	3805	int cpu;
3343	3806
3344	3807	*total = 0;
3345	3808	*entries = 0;
3346	3809
3347	3810	for_each_tracing_cpu(cpu) {
3348		- count = ring_buffer_entries_cpu(buf->buffer, cpu);
3349		- /*
3350		- * If this buffer has skipped entries, then we hold all
3351		- * entries for the trace and we need to ignore the
3352		- * ones before the time stamp.
3353		- */
3354		- if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3355		- count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3356		- /* total is the same as the entries */
3357		- *total += count;
3358		- } else
3359		- *total += count +
3360		- ring_buffer_overrun_cpu(buf->buffer, cpu);
3361		- *entries += count;
	3811	+ get_total_entries_cpu(buf, &t, &e, cpu);
	3812	+ *total += t;
	3813	+ *entries += e;
3362	3814	}
	3815	+}
	3816	+
	3817	+unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
	3818	+{
	3819	+ unsigned long total, entries;
	3820	+
	3821	+ if (!tr)
	3822	+ tr = &global_trace;
	3823	+
	3824	+ get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
	3825	+
	3826	+ return entries;
	3827	+}
	3828	+
	3829	+unsigned long trace_total_entries(struct trace_array *tr)
	3830	+{
	3831	+ unsigned long total, entries;
	3832	+
	3833	+ if (!tr)
	3834	+ tr = &global_trace;
	3835	+
	3836	+ get_total_entries(&tr->array_buffer, &total, &entries);
	3837	+
	3838	+ return entries;
3363	3839	}
3364	3840
3365	3841	static void print_lat_help_header(struct seq_file *m)
3366	3842	{
3367		- seq_puts(m, "# _--------=> CPU# \n"
3368		- "# / _-------=> irqs-off \n"
3369		- "# \| / _------=> need-resched \n"
3370		- "# \|\| / _-----=> need-resched_lazy \n"
3371		- "# \|\|\| / _----=> hardirq/softirq \n"
3372		- "# \|\|\|\| / _---=> preempt-depth \n"
3373		- "# \|\|\|\|\| / _--=> preempt-lazy-depth\n"
3374		- "# \|\|\|\|\|\| / _-=> migrate-disable \n"
3375		- "# \|\|\|\|\|\|\| / delay \n"
3376		- "# cmd pid \|\|\|\|\|\|\|\| time \| caller \n"
3377		- "# \\ / \|\|\|\|\|\|\|\| \\ \| / \n");
	3843	+ seq_puts(m, "# _--------=> CPU# \n"
	3844	+ "# / _-------=> irqs-off \n"
	3845	+ "# \| / _------=> need-resched \n"
	3846	+ "# \|\| / _-----=> need-resched-lazy\n"
	3847	+ "# \|\|\| / _----=> hardirq/softirq \n"
	3848	+ "# \|\|\|\| / _---=> preempt-depth \n"
	3849	+ "# \|\|\|\|\| / _--=> preempt-lazy-depth\n"
	3850	+ "# \|\|\|\|\|\| / _-=> migrate-disable \n"
	3851	+ "# \|\|\|\|\|\|\| / delay \n"
	3852	+ "# cmd pid \|\|\|\|\|\|\|\| time \| caller \n"
	3853	+ "# \\ / \|\|\|\|\|\|\|\| \\ \| / \n");
3378	3854	}
3379	3855
3380		-static void print_event_info(struct trace_buffer buf, struct seq_file m)
	3856	+static void print_event_info(struct array_buffer buf, struct seq_file m)
3381	3857	{
3382	3858	unsigned long total;
3383	3859	unsigned long entries;
..	..	@@ -3388,49 +3864,43 @@
3388	3864	seq_puts(m, "#\n");
3389	3865	}
3390	3866
3391		-static void print_func_help_header(struct trace_buffer buf, struct seq_file m,
	3867	+static void print_func_help_header(struct array_buffer buf, struct seq_file m,
3392	3868	unsigned int flags)
3393	3869	{
3394	3870	bool tgid = flags & TRACE_ITER_RECORD_TGID;
3395	3871
3396	3872	print_event_info(buf, m);
3397	3873
3398		- seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? "TGID " : "");
3399		- seq_printf(m, "# \| \| %s \| \| \|\n", tgid ? " \| " : "");
	3874	+ seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "");
	3875	+ seq_printf(m, "# \| \| %s \| \| \|\n", tgid ? " \| " : "");
3400	3876	}
3401	3877
3402		-static void print_func_help_header_irq(struct trace_buffer buf, struct seq_file m,
	3878	+static void print_func_help_header_irq(struct array_buffer buf, struct seq_file m,
3403	3879	unsigned int flags)
3404	3880	{
3405	3881	bool tgid = flags & TRACE_ITER_RECORD_TGID;
3406		- const char tgid_space[] = " ";
3407		- const char space[] = " ";
	3882	+ const char *space = " ";
	3883	+ int prec = tgid ? 12 : 2;
3408	3884
3409	3885	print_event_info(buf, m);
3410	3886
3411		- seq_printf(m, "# %s _-----=> irqs-off\n",
3412		- tgid ? tgid_space : space);
3413		- seq_printf(m, "# %s / _----=> need-resched\n",
3414		- tgid ? tgid_space : space);
3415		- seq_printf(m, "# %s\| / _---=> need-resched_lazy\n",
3416		- tgid ? tgid_space : space);
3417		- seq_printf(m, "# %s\|\| / _--=> hardirq/softirq\n",
3418		- tgid ? tgid_space : space);
3419		- seq_printf(m, "# %s\|\|\| / preempt-depth\n",
3420		- tgid ? tgid_space : space);
3421		- seq_printf(m, "# %s\|\|\|\| / delay\n",
3422		- tgid ? tgid_space : space);
3423		- seq_printf(m, "# TASK-PID %sCPU# \|\|\|\|\| TIMESTAMP FUNCTION\n",
3424		- tgid ? " TGID " : space);
3425		- seq_printf(m, "# \| \| %s \| \|\|\|\|\| \| \|\n",
3426		- tgid ? " \| " : space);
	3887	+ seq_printf(m, "# %.*s _-------=> irqs-off\n", prec, space);
	3888	+ seq_printf(m, "# %.*s / _------=> need-resched\n", prec, space);
	3889	+ seq_printf(m, "# %.*s\| / _-----=> need-resched-lazy\n", prec, space);
	3890	+ seq_printf(m, "# %.*s\|\| / _----=> hardirq/softirq\n", prec, space);
	3891	+ seq_printf(m, "# %.*s\|\|\| / _---=> preempt-depth\n", prec, space);
	3892	+ seq_printf(m, "# %.*s\|\|\|\| / _--=> preempt-lazy-depth\n", prec, space);
	3893	+ seq_printf(m, "# %.*s\|\|\|\|\| / _-=> migrate-disable\n", prec, space);
	3894	+ seq_printf(m, "# %.*s\|\|\|\|\|\| / delay\n", prec, space);
	3895	+ seq_printf(m, "# TASK-PID %.*s CPU# \|\|\|\|\|\|\| TIMESTAMP FUNCTION\n", prec, " TGID ");
	3896	+ seq_printf(m, "# \| \| %.*s \| \|\|\|\|\|\|\| \| \|\n", prec, " \| ");
3427	3897	}
3428	3898
3429	3899	void
3430	3900	print_trace_header(struct seq_file m, struct trace_iterator iter)
3431	3901	{
3432	3902	unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
3433		- struct trace_buffer *buf = iter->trace_buffer;
	3903	+ struct array_buffer *buf = iter->array_buffer;
3434	3904	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
3435	3905	struct tracer *type = iter->trace;
3436	3906	unsigned long entries;
..	..	@@ -3457,6 +3927,8 @@
3457	3927	"desktop",
3458	3928	#elif defined(CONFIG_PREEMPT)
3459	3929	"preempt",
	3930	+#elif defined(CONFIG_PREEMPT_RT)
	3931	+ "preempt_rt",
3460	3932	#else
3461	3933	"unknown",
3462	3934	#endif
..	..	@@ -3503,7 +3975,7 @@
3503	3975	cpumask_test_cpu(iter->cpu, iter->started))
3504	3976	return;
3505	3977
3506		- if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
	3978	+ if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
3507	3979	return;
3508	3980
3509	3981	if (cpumask_available(iter->started))
..	..	@@ -3637,7 +4109,7 @@
3637	4109	if (!ring_buffer_iter_empty(buf_iter))
3638	4110	return 0;
3639	4111	} else {
3640		- if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
	4112	+ if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
3641	4113	return 0;
3642	4114	}
3643	4115	return 1;
..	..	@@ -3649,7 +4121,7 @@
3649	4121	if (!ring_buffer_iter_empty(buf_iter))
3650	4122	return 0;
3651	4123	} else {
3652		- if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
	4124	+ if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
3653	4125	return 0;
3654	4126	}
3655	4127	}
..	..	@@ -3665,8 +4137,12 @@
3665	4137	enum print_line_t ret;
3666	4138
3667	4139	if (iter->lost_events) {
3668		- trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3669		- iter->cpu, iter->lost_events);
	4140	+ if (iter->lost_events == (unsigned long)-1)
	4141	+ trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
	4142	+ iter->cpu);
	4143	+ else
	4144	+ trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
	4145	+ iter->cpu, iter->lost_events);
3670	4146	if (trace_seq_has_overflowed(&iter->seq))
3671	4147	return TRACE_TYPE_PARTIAL_LINE;
3672	4148	}
..	..	@@ -3739,10 +4215,10 @@
3739	4215	} else {
3740	4216	if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3741	4217	if (trace_flags & TRACE_ITER_IRQ_INFO)
3742		- print_func_help_header_irq(iter->trace_buffer,
	4218	+ print_func_help_header_irq(iter->array_buffer,
3743	4219	m, trace_flags);
3744	4220	else
3745		- print_func_help_header(iter->trace_buffer, m,
	4221	+ print_func_help_header(iter->array_buffer, m,
3746	4222	trace_flags);
3747	4223	}
3748	4224	}
..	..	@@ -3882,6 +4358,18 @@
3882	4358	goto release;
3883	4359
3884	4360	/*
	4361	+ * trace_find_next_entry() may need to save off iter->ent.
	4362	+ * It will place it into the iter->temp buffer. As most
	4363	+ * events are less than 128, allocate a buffer of that size.
	4364	+ * If one is greater, then trace_find_next_entry() will
	4365	+ * allocate a new buffer to adjust for the bigger iter->ent.
	4366	+ * It's not critical if it fails to get allocated here.
	4367	+ */
	4368	+ iter->temp = kmalloc(128, GFP_KERNEL);
	4369	+ if (iter->temp)
	4370	+ iter->temp_size = 128;
	4371	+
	4372	+ /*
3885	4373	* We make a copy of the current tracer to avoid concurrent
3886	4374	* changes on it while we are reading.
3887	4375	*/
..	..	@@ -3900,35 +4388,38 @@
3900	4388	#ifdef CONFIG_TRACER_MAX_TRACE
3901	4389	/* Currently only the top directory has a snapshot */
3902	4390	if (tr->current_trace->print_max \|\| snapshot)
3903		- iter->trace_buffer = &tr->max_buffer;
	4391	+ iter->array_buffer = &tr->max_buffer;
3904	4392	else
3905	4393	#endif
3906		- iter->trace_buffer = &tr->trace_buffer;
	4394	+ iter->array_buffer = &tr->array_buffer;
3907	4395	iter->snapshot = snapshot;
3908	4396	iter->pos = -1;
3909	4397	iter->cpu_file = tracing_get_cpu(inode);
3910	4398	mutex_init(&iter->mutex);
3911	4399
3912	4400	/* Notify the tracer early; before we stop tracing. */
3913		- if (iter->trace && iter->trace->open)
	4401	+ if (iter->trace->open)
3914	4402	iter->trace->open(iter);
3915	4403
3916	4404	/* Annotate start of buffers if we had overruns */
3917		- if (ring_buffer_overruns(iter->trace_buffer->buffer))
	4405	+ if (ring_buffer_overruns(iter->array_buffer->buffer))
3918	4406	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
3919	4407
3920	4408	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
3921	4409	if (trace_clocks[tr->clock_id].in_ns)
3922	4410	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
3923	4411
3924		- /* stop the trace while dumping if we are not opening "snapshot" */
3925		- if (!iter->snapshot)
	4412	+ /*
	4413	+ * If pause-on-trace is enabled, then stop the trace while
	4414	+ * dumping, unless this is the "snapshot" file
	4415	+ */
	4416	+ if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
3926	4417	tracing_stop_tr(tr);
3927	4418
3928	4419	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3929	4420	for_each_tracing_cpu(cpu) {
3930	4421	iter->buffer_iter[cpu] =
3931		- ring_buffer_read_prepare(iter->trace_buffer->buffer,
	4422	+ ring_buffer_read_prepare(iter->array_buffer->buffer,
3932	4423	cpu, GFP_KERNEL);
3933	4424	}
3934	4425	ring_buffer_read_prepare_sync();
..	..	@@ -3939,7 +4430,7 @@
3939	4430	} else {
3940	4431	cpu = iter->cpu_file;
3941	4432	iter->buffer_iter[cpu] =
3942		- ring_buffer_read_prepare(iter->trace_buffer->buffer,
	4433	+ ring_buffer_read_prepare(iter->array_buffer->buffer,
3943	4434	cpu, GFP_KERNEL);
3944	4435	ring_buffer_read_prepare_sync();
3945	4436	ring_buffer_read_start(iter->buffer_iter[cpu]);
..	..	@@ -3953,6 +4444,7 @@
3953	4444	fail:
3954	4445	mutex_unlock(&trace_types_lock);
3955	4446	kfree(iter->trace);
	4447	+ kfree(iter->temp);
3956	4448	kfree(iter->buffer_iter);
3957	4449	release:
3958	4450	seq_release_private(inode, file);
..	..	@@ -3961,8 +4453,11 @@
3961	4453
3962	4454	int tracing_open_generic(struct inode inode, struct file filp)
3963	4455	{
3964		- if (tracing_disabled)
3965		- return -ENODEV;
	4456	+ int ret;
	4457	+
	4458	+ ret = tracing_check_open_get_tr(NULL);
	4459	+ if (ret)
	4460	+ return ret;
3966	4461
3967	4462	filp->private_data = inode->i_private;
3968	4463	return 0;
..	..	@@ -3977,15 +4472,14 @@
3977	4472	* Open and update trace_array ref count.
3978	4473	* Must have the current trace_array passed to it.
3979	4474	*/
3980		-static int tracing_open_generic_tr(struct inode inode, struct file filp)
	4475	+int tracing_open_generic_tr(struct inode inode, struct file filp)
3981	4476	{
3982	4477	struct trace_array *tr = inode->i_private;
	4478	+ int ret;
3983	4479
3984		- if (tracing_disabled)
3985		- return -ENODEV;
3986		-
3987		- if (trace_array_get(tr) < 0)
3988		- return -ENODEV;
	4480	+ ret = tracing_check_open_get_tr(tr);
	4481	+ if (ret)
	4482	+ return ret;
3989	4483
3990	4484	filp->private_data = inode->i_private;
3991	4485
..	..	@@ -4016,7 +4510,7 @@
4016	4510	if (iter->trace && iter->trace->close)
4017	4511	iter->trace->close(iter);
4018	4512
4019		- if (!iter->snapshot)
	4513	+ if (!iter->snapshot && tr->stop_count)
4020	4514	/* reenable tracing if it was previously enabled */
4021	4515	tracing_start_tr(tr);
4022	4516
..	..	@@ -4026,6 +4520,7 @@
4026	4520
4027	4521	mutex_destroy(&iter->mutex);
4028	4522	free_cpumask_var(iter->started);
	4523	+ kfree(iter->temp);
4029	4524	kfree(iter->trace);
4030	4525	kfree(iter->buffer_iter);
4031	4526	seq_release_private(inode, file);
..	..	@@ -4054,15 +4549,16 @@
4054	4549	{
4055	4550	struct trace_array *tr = inode->i_private;
4056	4551	struct trace_iterator *iter;
4057		- int ret = 0;
	4552	+ int ret;
4058	4553
4059		- if (trace_array_get(tr) < 0)
4060		- return -ENODEV;
	4554	+ ret = tracing_check_open_get_tr(tr);
	4555	+ if (ret)
	4556	+ return ret;
4061	4557
4062	4558	/* If this file was open for write, then erase contents */
4063	4559	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4064	4560	int cpu = tracing_get_cpu(inode);
4065		- struct trace_buffer *trace_buf = &tr->trace_buffer;
	4561	+ struct array_buffer *trace_buf = &tr->array_buffer;
4066	4562
4067	4563	#ifdef CONFIG_TRACER_MAX_TRACE
4068	4564	if (tr->current_trace->print_max)
..	..	@@ -4072,7 +4568,7 @@
4072	4568	if (cpu == RING_BUFFER_ALL_CPUS)
4073	4569	tracing_reset_online_cpus(trace_buf);
4074	4570	else
4075		- tracing_reset(trace_buf, cpu);
	4571	+ tracing_reset_cpu(trace_buf, cpu);
4076	4572	}
4077	4573
4078	4574	if (file->f_mode & FMODE_READ) {
..	..	@@ -4173,11 +4669,9 @@
4173	4669	struct seq_file *m;
4174	4670	int ret;
4175	4671
4176		- if (tracing_disabled)
4177		- return -ENODEV;
4178		-
4179		- if (trace_array_get(tr) < 0)
4180		- return -ENODEV;
	4672	+ ret = tracing_check_open_get_tr(tr);
	4673	+ if (ret)
	4674	+ return ret;
4181	4675
4182	4676	ret = seq_open(file, &show_traces_seq_ops);
4183	4677	if (ret) {
..	..	@@ -4261,20 +4755,13 @@
4261	4755	return count;
4262	4756	}
4263	4757
4264		-static ssize_t
4265		-tracing_cpumask_write(struct file filp, const char __user ubuf,
4266		- size_t count, loff_t *ppos)
	4758	+int tracing_set_cpumask(struct trace_array *tr,
	4759	+ cpumask_var_t tracing_cpumask_new)
4267	4760	{
4268		- struct trace_array *tr = file_inode(filp)->i_private;
4269		- cpumask_var_t tracing_cpumask_new;
4270		- int err, cpu;
	4761	+ int cpu;
4271	4762
4272		- if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4273		- return -ENOMEM;
4274		-
4275		- err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
4276		- if (err)
4277		- goto err_unlock;
	4763	+ if (!tr)
	4764	+ return -EINVAL;
4278	4765
4279	4766	local_irq_disable();
4280	4767	arch_spin_lock(&tr->max_lock);
..	..	@@ -4285,24 +4772,47 @@
4285	4772	*/
4286	4773	if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4287	4774	!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4288		- atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4289		- ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
	4775	+ atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
	4776	+ ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
4290	4777	}
4291	4778	if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4292	4779	cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4293		- atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4294		- ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
	4780	+ atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
	4781	+ ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
4295	4782	}
4296	4783	}
4297	4784	arch_spin_unlock(&tr->max_lock);
4298	4785	local_irq_enable();
4299	4786
4300	4787	cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
	4788	+
	4789	+ return 0;
	4790	+}
	4791	+
	4792	+static ssize_t
	4793	+tracing_cpumask_write(struct file filp, const char __user ubuf,
	4794	+ size_t count, loff_t *ppos)
	4795	+{
	4796	+ struct trace_array *tr = file_inode(filp)->i_private;
	4797	+ cpumask_var_t tracing_cpumask_new;
	4798	+ int err;
	4799	+
	4800	+ if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
	4801	+ return -ENOMEM;
	4802	+
	4803	+ err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
	4804	+ if (err)
	4805	+ goto err_free;
	4806	+
	4807	+ err = tracing_set_cpumask(tr, tracing_cpumask_new);
	4808	+ if (err)
	4809	+ goto err_free;
	4810	+
4301	4811	free_cpumask_var(tracing_cpumask_new);
4302	4812
4303	4813	return count;
4304	4814
4305		-err_unlock:
	4815	+err_free:
4306	4816	free_cpumask_var(tracing_cpumask_new);
4307	4817
4308	4818	return err;
..	..	@@ -4444,7 +4954,7 @@
4444	4954	ftrace_pid_follow_fork(tr, enabled);
4445	4955
4446	4956	if (mask == TRACE_ITER_OVERWRITE) {
4447		- ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
	4957	+ ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
4448	4958	#ifdef CONFIG_TRACER_MAX_TRACE
4449	4959	ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
4450	4960	#endif
..	..	@@ -4458,19 +4968,21 @@
4458	4968	return 0;
4459	4969	}
4460	4970
4461		-static int trace_set_options(struct trace_array tr, char option)
	4971	+int trace_set_options(struct trace_array tr, char option)
4462	4972	{
4463	4973	char *cmp;
4464	4974	int neg = 0;
4465	4975	int ret;
4466	4976	size_t orig_len = strlen(option);
	4977	+ int len;
4467	4978
4468	4979	cmp = strstrip(option);
4469	4980
4470		- if (strncmp(cmp, "no", 2) == 0) {
	4981	+ len = str_has_prefix(cmp, "no");
	4982	+ if (len)
4471	4983	neg = 1;
4472		- cmp += 2;
4473		- }
	4984	+
	4985	+ cmp += len;
4474	4986
4475	4987	mutex_lock(&event_mutex);
4476	4988	mutex_lock(&trace_types_lock);
..	..	@@ -4546,11 +5058,9 @@
4546	5058	struct trace_array *tr = inode->i_private;
4547	5059	int ret;
4548	5060
4549		- if (tracing_disabled)
4550		- return -ENODEV;
4551		-
4552		- if (trace_array_get(tr) < 0)
4553		- return -ENODEV;
	5061	+ ret = tracing_check_open_get_tr(tr);
	5062	+ if (ret)
	5063	+ return ret;
4554	5064
4555	5065	ret = single_open(file, tracing_trace_options_show, inode->i_private);
4556	5066	if (ret < 0)
..	..	@@ -4577,6 +5087,7 @@
4577	5087	" trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4578	5088	" current_tracer\t- function and latency tracers\n"
4579	5089	" available_tracers\t- list of configured tracers for current_tracer\n"
	5090	+ " error_log\t- error log for failed commands (that support it)\n"
4580	5091	" buffer_size_kb\t- view and modify size of per cpu buffer\n"
4581	5092	" buffer_total_size_kb - view total size of all cpu buffers\n\n"
4582	5093	" trace_clock\t\t-change the clock used to order events\n"
..	..	@@ -4597,7 +5108,7 @@
4597	5108	" instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4598	5109	"\t\t\t Remove sub-buffer with rmdir\n"
4599	5110	" trace_options\t\t- Set format or modify how tracing happens\n"
4600		- "\t\t\t Disable an option by adding a suffix 'no' to the\n"
	5111	+ "\t\t\t Disable an option by prefixing 'no' to the\n"
4601	5112	"\t\t\t option name\n"
4602	5113	" saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4603	5114	#ifdef CONFIG_DYNAMIC_FTRACE
..	..	@@ -4641,6 +5152,8 @@
4641	5152	#ifdef CONFIG_FUNCTION_TRACER
4642	5153	" set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4643	5154	"\t\t (function)\n"
	5155	+ " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
	5156	+ "\t\t (function)\n"
4644	5157	#endif
4645	5158	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4646	5159	" set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
..	..	@@ -4662,31 +5175,49 @@
4662	5175	"\t\t\t traces\n"
4663	5176	#endif
4664	5177	#endif /* CONFIG_STACK_TRACER */
	5178	+#ifdef CONFIG_DYNAMIC_EVENTS
	5179	+ " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
	5180	+ "\t\t\t Write into this file to define/undefine new trace events.\n"
	5181	+#endif
4665	5182	#ifdef CONFIG_KPROBE_EVENTS
4666		- " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
	5183	+ " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
4667	5184	"\t\t\t Write into this file to define/undefine new trace events.\n"
4668	5185	#endif
4669	5186	#ifdef CONFIG_UPROBE_EVENTS
4670		- " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
	5187	+ " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
4671	5188	"\t\t\t Write into this file to define/undefine new trace events.\n"
4672	5189	#endif
4673	5190	#if defined(CONFIG_KPROBE_EVENTS) \|\| defined(CONFIG_UPROBE_EVENTS)
4674	5191	"\t accepts: event-definitions (one definition per line)\n"
4675	5192	"\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
4676	5193	"\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
	5194	+#ifdef CONFIG_HIST_TRIGGERS
	5195	+ "\t s:[synthetic/]<event> <field> [<field>]\n"
	5196	+#endif
4677	5197	"\t -:[<group>/]<event>\n"
4678	5198	#ifdef CONFIG_KPROBE_EVENTS
4679	5199	"\t place: [<module>:]<symbol>[+<offset>]\|<memaddr>\n"
4680		- "place (kretprobe): [<module>:]<symbol>[+<offset>]\|<memaddr>\n"
	5200	+ "place (kretprobe): [<module>:]<symbol>[+<offset>]%return\|<memaddr>\n"
4681	5201	#endif
4682	5202	#ifdef CONFIG_UPROBE_EVENTS
4683		- "\t place: <path>:<offset>\n"
	5203	+ " place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
4684	5204	#endif
4685	5205	"\t args: <name>=fetcharg[:type]\n"
4686	5206	"\t fetcharg: %<register>, @<address>, @<symbol>[+\|-<offset>],\n"
4687		- "\t $stack<index>, $stack, $retval, $comm\n"
4688		- "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n"
4689		- "\t b<bit-width>@<bit-offset>/<container-size>\n"
	5207	+#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
	5208	+ "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
	5209	+#else
	5210	+ "\t $stack<index>, $stack, $retval, $comm,\n"
	5211	+#endif
	5212	+ "\t +\|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
	5213	+ "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
	5214	+ "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
	5215	+ "\t <type>\\[<array-size>\\]\n"
	5216	+#ifdef CONFIG_HIST_TRIGGERS
	5217	+ "\t field: <stype> <name>;\n"
	5218	+ "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
	5219	+ "\t [unsigned] char/int/long\n"
	5220	+#endif
4690	5221	#endif
4691	5222	" events/\t\t- Directory containing all trace event subsystems:\n"
4692	5223	" enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
..	..	@@ -4739,6 +5270,7 @@
4739	5270	"\t [:size=#entries]\n"
4740	5271	"\t [:pause][:continue][:clear]\n"
4741	5272	"\t [:name=histname1]\n"
	5273	+ "\t [:<handler>.<action>]\n"
4742	5274	"\t [if <filter>]\n\n"
4743	5275	"\t Note, special fields can be used as well:\n"
4744	5276	"\t common_timestamp - to record current timestamp\n"
..	..	@@ -4783,8 +5315,26 @@
4783	5315	"\t unchanged.\n\n"
4784	5316	"\t The enable_hist and disable_hist triggers can be used to\n"
4785	5317	"\t have one event conditionally start and stop another event's\n"
4786		- "\t already-attached hist trigger. The syntax is analagous to\n"
4787		- "\t the enable_event and disable_event triggers.\n"
	5318	+ "\t already-attached hist trigger. The syntax is analogous to\n"
	5319	+ "\t the enable_event and disable_event triggers.\n\n"
	5320	+ "\t Hist trigger handlers and actions are executed whenever a\n"
	5321	+ "\t a histogram entry is added or updated. They take the form:\n\n"
	5322	+ "\t <handler>.<action>\n\n"
	5323	+ "\t The available handlers are:\n\n"
	5324	+ "\t onmatch(matching.event) - invoke on addition or update\n"
	5325	+ "\t onmax(var) - invoke if var exceeds current max\n"
	5326	+ "\t onchange(var) - invoke action if var changes\n\n"
	5327	+ "\t The available actions are:\n\n"
	5328	+ "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
	5329	+ "\t save(field,...) - save current event fields\n"
	5330	+#ifdef CONFIG_TRACER_SNAPSHOT
	5331	+ "\t snapshot() - snapshot the trace buffer\n\n"
	5332	+#endif
	5333	+#ifdef CONFIG_SYNTH_EVENTS
	5334	+ " events/synthetic_events\t- Create/append/remove/show synthetic events\n"
	5335	+ "\t Write into this file to define/undefine new synthetic events.\n"
	5336	+ "\t example: echo 'myevent u64 lat; char name[]' >> synthetic_events\n"
	5337	+#endif
4788	5338	#endif
4789	5339	;
4790	5340
..	..	@@ -4842,8 +5392,11 @@
4842	5392
4843	5393	static int tracing_saved_tgids_open(struct inode inode, struct file filp)
4844	5394	{
4845		- if (tracing_disabled)
4846		- return -ENODEV;
	5395	+ int ret;
	5396	+
	5397	+ ret = tracing_check_open_get_tr(NULL);
	5398	+ if (ret)
	5399	+ return ret;
4847	5400
4848	5401	return seq_open(filp, &tracing_saved_tgids_seq_ops);
4849	5402	}
..	..	@@ -4919,8 +5472,11 @@
4919	5472
4920	5473	static int tracing_saved_cmdlines_open(struct inode inode, struct file filp)
4921	5474	{
4922		- if (tracing_disabled)
4923		- return -ENODEV;
	5475	+ int ret;
	5476	+
	5477	+ ret = tracing_check_open_get_tr(NULL);
	5478	+ if (ret)
	5479	+ return ret;
4924	5480
4925	5481	return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
4926	5482	}
..	..	@@ -4939,9 +5495,11 @@
4939	5495	char buf[64];
4940	5496	int r;
4941	5497
	5498	+ preempt_disable();
4942	5499	arch_spin_lock(&trace_cmdline_lock);
4943	5500	r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
4944	5501	arch_spin_unlock(&trace_cmdline_lock);
	5502	+ preempt_enable();
4945	5503
4946	5504	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4947	5505	}
..	..	@@ -4966,10 +5524,12 @@
4966	5524	return -ENOMEM;
4967	5525	}
4968	5526
	5527	+ preempt_disable();
4969	5528	arch_spin_lock(&trace_cmdline_lock);
4970	5529	savedcmd_temp = savedcmd;
4971	5530	savedcmd = s;
4972	5531	arch_spin_unlock(&trace_cmdline_lock);
	5532	+ preempt_enable();
4973	5533	free_saved_cmdlines_buffer(savedcmd_temp);
4974	5534
4975	5535	return 0;
..	..	@@ -5028,14 +5588,12 @@
5028	5588	* Paranoid! If ptr points to end, we don't want to increment past it.
5029	5589	* This really should never happen.
5030	5590	*/
	5591	+ (*pos)++;
5031	5592	ptr = update_eval_map(ptr);
5032	5593	if (WARN_ON_ONCE(!ptr))
5033	5594	return NULL;
5034	5595
5035	5596	ptr++;
5036		-
5037		- (*pos)++;
5038		-
5039	5597	ptr = update_eval_map(ptr);
5040	5598
5041	5599	return ptr;
..	..	@@ -5084,8 +5642,11 @@
5084	5642
5085	5643	static int tracing_eval_map_open(struct inode inode, struct file filp)
5086	5644	{
5087		- if (tracing_disabled)
5088		- return -ENODEV;
	5645	+ int ret;
	5646	+
	5647	+ ret = tracing_check_open_get_tr(NULL);
	5648	+ if (ret)
	5649	+ return ret;
5089	5650
5090	5651	return seq_open(filp, &tracing_eval_map_seq_ops);
5091	5652	}
..	..	@@ -5198,11 +5759,11 @@
5198	5759
5199	5760	int tracer_init(struct tracer t, struct trace_array tr)
5200	5761	{
5201		- tracing_reset_online_cpus(&tr->trace_buffer);
	5762	+ tracing_reset_online_cpus(&tr->array_buffer);
5202	5763	return t->init(tr);
5203	5764	}
5204	5765
5205		-static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
	5766	+static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
5206	5767	{
5207	5768	int cpu;
5208	5769
..	..	@@ -5212,8 +5773,8 @@
5212	5773
5213	5774	#ifdef CONFIG_TRACER_MAX_TRACE
5214	5775	/* resize @tr's buffer to the size of @size_tr's entries */
5215		-static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
5216		- struct trace_buffer *size_buf, int cpu_id)
	5776	+static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
	5777	+ struct array_buffer *size_buf, int cpu_id)
5217	5778	{
5218	5779	int cpu, ret = 0;
5219	5780
..	..	@@ -5251,10 +5812,10 @@
5251	5812	ring_buffer_expanded = true;
5252	5813
5253	5814	/* May be called before buffers are initialized */
5254		- if (!tr->trace_buffer.buffer)
	5815	+ if (!tr->array_buffer.buffer)
5255	5816	return 0;
5256	5817
5257		- ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
	5818	+ ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
5258	5819	if (ret < 0)
5259	5820	return ret;
5260	5821
..	..	@@ -5265,8 +5826,8 @@
5265	5826
5266	5827	ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5267	5828	if (ret < 0) {
5268		- int r = resize_buffer_duplicate_size(&tr->trace_buffer,
5269		- &tr->trace_buffer, cpu);
	5829	+ int r = resize_buffer_duplicate_size(&tr->array_buffer,
	5830	+ &tr->array_buffer, cpu);
5270	5831	if (r < 0) {
5271	5832	/*
5272	5833	* AARGH! We are left with different
..	..	@@ -5297,15 +5858,15 @@
5297	5858	#endif /* CONFIG_TRACER_MAX_TRACE */
5298	5859
5299	5860	if (cpu == RING_BUFFER_ALL_CPUS)
5300		- set_buffer_entries(&tr->trace_buffer, size);
	5861	+ set_buffer_entries(&tr->array_buffer, size);
5301	5862	else
5302		- per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
	5863	+ per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size;
5303	5864
5304	5865	return ret;
5305	5866	}
5306	5867
5307		-static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5308		- unsigned long size, int cpu_id)
	5868	+ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
	5869	+ unsigned long size, int cpu_id)
5309	5870	{
5310	5871	int ret = size;
5311	5872
..	..	@@ -5375,16 +5936,22 @@
5375	5936	tr->current_trace = &nop_trace;
5376	5937	}
5377	5938
	5939	+static bool tracer_options_updated;
	5940	+
5378	5941	static void add_tracer_options(struct trace_array tr, struct tracer t)
5379	5942	{
5380	5943	/* Only enable if the directory has been created already. */
5381	5944	if (!tr->dir)
5382	5945	return;
5383	5946
	5947	+ /* Only create trace option files after update_tracer_options finish */
	5948	+ if (!tracer_options_updated)
	5949	+ return;
	5950	+
5384	5951	create_trace_option_files(tr, t);
5385	5952	}
5386	5953
5387		-static int tracing_set_tracer(struct trace_array tr, const char buf)
	5954	+int tracing_set_tracer(struct trace_array tr, const char buf)
5388	5955	{
5389	5956	struct tracer *t;
5390	5957	#ifdef CONFIG_TRACER_MAX_TRACE
..	..	@@ -5413,6 +5980,18 @@
5413	5980	if (t == tr->current_trace)
5414	5981	goto out;
5415	5982
	5983	+#ifdef CONFIG_TRACER_SNAPSHOT
	5984	+ if (t->use_max_tr) {
	5985	+ local_irq_disable();
	5986	+ arch_spin_lock(&tr->max_lock);
	5987	+ if (tr->cond_snapshot)
	5988	+ ret = -EBUSY;
	5989	+ arch_spin_unlock(&tr->max_lock);
	5990	+ local_irq_enable();
	5991	+ if (ret)
	5992	+ goto out;
	5993	+ }
	5994	+#endif
5416	5995	/* Some tracers won't work on kernel command line */
5417	5996	if (system_state < SYSTEM_RUNNING && t->noboot) {
5418	5997	pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
..	..	@@ -5427,7 +6006,7 @@
5427	6006	}
5428	6007
5429	6008	/* If trace pipe files are being read, we can't change the tracer */
5430		- if (tr->current_trace->ref) {
	6009	+ if (tr->trace_ref) {
5431	6010	ret = -EBUSY;
5432	6011	goto out;
5433	6012	}
..	..	@@ -5439,11 +6018,11 @@
5439	6018	if (tr->current_trace->reset)
5440	6019	tr->current_trace->reset(tr);
5441	6020
5442		- /* Current trace needs to be nop_trace before synchronize_sched */
5443		- tr->current_trace = &nop_trace;
5444		-
5445	6021	#ifdef CONFIG_TRACER_MAX_TRACE
5446		- had_max_tr = tr->allocated_snapshot;
	6022	+ had_max_tr = tr->current_trace->use_max_tr;
	6023	+
	6024	+ /* Current trace needs to be nop_trace before synchronize_rcu */
	6025	+ tr->current_trace = &nop_trace;
5447	6026
5448	6027	if (had_max_tr && !t->use_max_tr) {
5449	6028	/*
..	..	@@ -5453,17 +6032,17 @@
5453	6032	* The update_max_tr is called from interrupts disabled
5454	6033	* so a synchronized_sched() is sufficient.
5455	6034	*/
5456		- synchronize_sched();
	6035	+ synchronize_rcu();
5457	6036	free_snapshot(tr);
5458	6037	}
5459		-#endif
5460	6038
5461		-#ifdef CONFIG_TRACER_MAX_TRACE
5462		- if (t->use_max_tr && !had_max_tr) {
	6039	+ if (t->use_max_tr && !tr->allocated_snapshot) {
5463	6040	ret = tracing_alloc_snapshot_instance(tr);
5464	6041	if (ret < 0)
5465	6042	goto out;
5466	6043	}
	6044	+#else
	6045	+ tr->current_trace = &nop_trace;
5467	6046	#endif
5468	6047
5469	6048	if (t->init) {
..	..	@@ -5598,13 +6177,11 @@
5598	6177	{
5599	6178	struct trace_array *tr = inode->i_private;
5600	6179	struct trace_iterator *iter;
5601		- int ret = 0;
	6180	+ int ret;
5602	6181
5603		- if (tracing_disabled)
5604		- return -ENODEV;
5605		-
5606		- if (trace_array_get(tr) < 0)
5607		- return -ENODEV;
	6182	+ ret = tracing_check_open_get_tr(tr);
	6183	+ if (ret)
	6184	+ return ret;
5608	6185
5609	6186	mutex_lock(&trace_types_lock);
5610	6187
..	..	@@ -5635,7 +6212,7 @@
5635	6212	iter->iter_flags \|= TRACE_FILE_TIME_IN_NS;
5636	6213
5637	6214	iter->tr = tr;
5638		- iter->trace_buffer = &tr->trace_buffer;
	6215	+ iter->array_buffer = &tr->array_buffer;
5639	6216	iter->cpu_file = tracing_get_cpu(inode);
5640	6217	mutex_init(&iter->mutex);
5641	6218	filp->private_data = iter;
..	..	@@ -5645,7 +6222,7 @@
5645	6222
5646	6223	nonseekable_open(inode, filp);
5647	6224
5648		- tr->current_trace->ref++;
	6225	+ tr->trace_ref++;
5649	6226	out:
5650	6227	mutex_unlock(&trace_types_lock);
5651	6228	return ret;
..	..	@@ -5664,7 +6241,7 @@
5664	6241
5665	6242	mutex_lock(&trace_types_lock);
5666	6243
5667		- tr->current_trace->ref--;
	6244	+ tr->trace_ref--;
5668	6245
5669	6246	if (iter->trace->pipe_close)
5670	6247	iter->trace->pipe_close(iter);
..	..	@@ -5695,8 +6272,8 @@
5695	6272	*/
5696	6273	return EPOLLIN \| EPOLLRDNORM;
5697	6274	else
5698		- return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
5699		- filp, poll_table);
	6275	+ return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
	6276	+ filp, poll_table, iter->tr->buffer_percent);
5700	6277	}
5701	6278
5702	6279	static __poll_t
..	..	@@ -5733,7 +6310,7 @@
5733	6310
5734	6311	mutex_unlock(&iter->mutex);
5735	6312
5736		- ret = wait_on_pipe(iter, false);
	6313	+ ret = wait_on_pipe(iter, 0);
5737	6314
5738	6315	mutex_lock(&iter->mutex);
5739	6316
..	..	@@ -5849,14 +6426,6 @@
5849	6426	__free_page(spd->pages[idx]);
5850	6427	}
5851	6428
5852		-static const struct pipe_buf_operations tracing_pipe_buf_ops = {
5853		- .can_merge = 0,
5854		- .confirm = generic_pipe_buf_confirm,
5855		- .release = generic_pipe_buf_release,
5856		- .steal = generic_pipe_buf_steal,
5857		- .get = generic_pipe_buf_get,
5858		-};
5859		-
5860	6429	static size_t
5861	6430	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
5862	6431	{
..	..	@@ -5918,7 +6487,7 @@
5918	6487	.partial = partial_def,
5919	6488	.nr_pages = 0, /* This gets updated below. */
5920	6489	.nr_pages_max = PIPE_DEF_BUFFERS,
5921		- .ops = &tracing_pipe_buf_ops,
	6490	+ .ops = &default_pipe_buf_ops,
5922	6491	.spd_release = tracing_spd_release_pipe,
5923	6492	};
5924	6493	ssize_t ret;
..	..	@@ -6013,8 +6582,8 @@
6013	6582	for_each_tracing_cpu(cpu) {
6014	6583	/* fill in the size from first enabled cpu */
6015	6584	if (size == 0)
6016		- size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
6017		- if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
	6585	+ size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
	6586	+ if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
6018	6587	buf_size_same = 0;
6019	6588	break;
6020	6589	}
..	..	@@ -6030,7 +6599,7 @@
6030	6599	} else
6031	6600	r = sprintf(buf, "X\n");
6032	6601	} else
6033		- r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
	6602	+ r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
6034	6603
6035	6604	mutex_unlock(&trace_types_lock);
6036	6605
..	..	@@ -6077,7 +6646,7 @@
6077	6646
6078	6647	mutex_lock(&trace_types_lock);
6079	6648	for_each_tracing_cpu(cpu) {
6080		- size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
	6649	+ size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
6081	6650	if (!ring_buffer_expanded)
6082	6651	expanded_size += trace_buf_size >> 10;
6083	6652	}
..	..	@@ -6127,16 +6696,15 @@
6127	6696	struct trace_array *tr = filp->private_data;
6128	6697	struct ring_buffer_event *event;
6129	6698	enum event_trigger_type tt = ETT_NONE;
6130		- struct ring_buffer *buffer;
	6699	+ struct trace_buffer *buffer;
6131	6700	struct print_entry *entry;
6132		- unsigned long irq_flags;
6133		- const char faulted[] = "<faulted>";
6134	6701	ssize_t written;
6135	6702	int size;
6136	6703	int len;
6137	6704
6138	6705	/* Used in tracing_mark_raw_write() as well */
6139		-#define FAULTED_SIZE (sizeof(faulted) - 1) /* '\0' is already accounted for */
	6706	+#define FAULTED_STR "<faulted>"
	6707	+#define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
6140	6708
6141	6709	if (tracing_disabled)
6142	6710	return -EINVAL;
..	..	@@ -6149,16 +6717,15 @@
6149	6717
6150	6718	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6151	6719
6152		- local_save_flags(irq_flags);
6153	6720	size = sizeof(entry) + cnt + 2; / add '\0' and possible '\n' */
6154	6721
6155	6722	/* If less than "<faulted>", then make sure we can still add that */
6156	6723	if (cnt < FAULTED_SIZE)
6157	6724	size += FAULTED_SIZE - cnt;
6158	6725
6159		- buffer = tr->trace_buffer.buffer;
	6726	+ buffer = tr->array_buffer.buffer;
6160	6727	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6161		- irq_flags, preempt_count());
	6728	+ tracing_gen_ctx());
6162	6729	if (unlikely(!event))
6163	6730	/* Ring buffer disabled, return as if not open for write */
6164	6731	return -EBADF;
..	..	@@ -6168,12 +6735,11 @@
6168	6735
6169	6736	len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6170	6737	if (len) {
6171		- memcpy(&entry->buf, faulted, FAULTED_SIZE);
	6738	+ memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6172	6739	cnt = FAULTED_SIZE;
6173	6740	written = -EFAULT;
6174	6741	} else
6175	6742	written = cnt;
6176		- len = cnt;
6177	6743
6178	6744	if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
6179	6745	/* do not add \n before testing triggers, but add \0 */
..	..	@@ -6187,6 +6753,8 @@
6187	6753	} else
6188	6754	entry->buf[cnt] = '\0';
6189	6755
	6756	+ if (static_branch_unlikely(&trace_marker_exports_enabled))
	6757	+ ftrace_exports(event, TRACE_EXPORT_MARKER);
6190	6758	__buffer_unlock_commit(buffer, event);
6191	6759
6192	6760	if (tt)
..	..	@@ -6207,10 +6775,8 @@
6207	6775	{
6208	6776	struct trace_array *tr = filp->private_data;
6209	6777	struct ring_buffer_event *event;
6210		- struct ring_buffer *buffer;
	6778	+ struct trace_buffer *buffer;
6211	6779	struct raw_data_entry *entry;
6212		- const char faulted[] = "<faulted>";
6213		- unsigned long irq_flags;
6214	6780	ssize_t written;
6215	6781	int size;
6216	6782	int len;
..	..	@@ -6232,14 +6798,13 @@
6232	6798
6233	6799	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6234	6800
6235		- local_save_flags(irq_flags);
6236	6801	size = sizeof(*entry) + cnt;
6237	6802	if (cnt < FAULT_SIZE_ID)
6238	6803	size += FAULT_SIZE_ID - cnt;
6239	6804
6240		- buffer = tr->trace_buffer.buffer;
	6805	+ buffer = tr->array_buffer.buffer;
6241	6806	event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6242		- irq_flags, preempt_count());
	6807	+ tracing_gen_ctx());
6243	6808	if (!event)
6244	6809	/* Ring buffer disabled, return as if not open for write */
6245	6810	return -EBADF;
..	..	@@ -6249,7 +6814,7 @@
6249	6814	len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6250	6815	if (len) {
6251	6816	entry->id = -1;
6252		- memcpy(&entry->buf, faulted, FAULTED_SIZE);
	6817	+ memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6253	6818	written = -EFAULT;
6254	6819	} else
6255	6820	written = cnt;
..	..	@@ -6292,13 +6857,13 @@
6292	6857
6293	6858	tr->clock_id = i;
6294	6859
6295		- ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
	6860	+ ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
6296	6861
6297	6862	/*
6298	6863	* New clock may not be consistent with the previous clock.
6299	6864	* Reset the buffer so that it doesn't have incomparable timestamps.
6300	6865	*/
6301		- tracing_reset_online_cpus(&tr->trace_buffer);
	6866	+ tracing_reset_online_cpus(&tr->array_buffer);
6302	6867
6303	6868	#ifdef CONFIG_TRACER_MAX_TRACE
6304	6869	if (tr->max_buffer.buffer)
..	..	@@ -6344,11 +6909,9 @@
6344	6909	struct trace_array *tr = inode->i_private;
6345	6910	int ret;
6346	6911
6347		- if (tracing_disabled)
6348		- return -ENODEV;
6349		-
6350		- if (trace_array_get(tr))
6351		- return -ENODEV;
	6912	+ ret = tracing_check_open_get_tr(tr);
	6913	+ if (ret)
	6914	+ return ret;
6352	6915
6353	6916	ret = single_open(file, tracing_clock_show, inode->i_private);
6354	6917	if (ret < 0)
..	..	@@ -6363,7 +6926,7 @@
6363	6926
6364	6927	mutex_lock(&trace_types_lock);
6365	6928
6366		- if (ring_buffer_time_stamp_abs(tr->trace_buffer.buffer))
	6929	+ if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
6367	6930	seq_puts(m, "delta [absolute]\n");
6368	6931	else
6369	6932	seq_puts(m, "[delta] absolute\n");
..	..	@@ -6378,11 +6941,9 @@
6378	6941	struct trace_array *tr = inode->i_private;
6379	6942	int ret;
6380	6943
6381		- if (tracing_disabled)
6382		- return -ENODEV;
6383		-
6384		- if (trace_array_get(tr))
6385		- return -ENODEV;
	6944	+ ret = tracing_check_open_get_tr(tr);
	6945	+ if (ret)
	6946	+ return ret;
6386	6947
6387	6948	ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6388	6949	if (ret < 0)
..	..	@@ -6410,7 +6971,7 @@
6410	6971	goto out;
6411	6972	}
6412	6973
6413		- ring_buffer_set_time_stamp_abs(tr->trace_buffer.buffer, abs);
	6974	+ ring_buffer_set_time_stamp_abs(tr->array_buffer.buffer, abs);
6414	6975
6415	6976	#ifdef CONFIG_TRACER_MAX_TRACE
6416	6977	if (tr->max_buffer.buffer)
..	..	@@ -6435,10 +6996,11 @@
6435	6996	struct trace_array *tr = inode->i_private;
6436	6997	struct trace_iterator *iter;
6437	6998	struct seq_file *m;
6438		- int ret = 0;
	6999	+ int ret;
6439	7000
6440		- if (trace_array_get(tr) < 0)
6441		- return -ENODEV;
	7001	+ ret = tracing_check_open_get_tr(tr);
	7002	+ if (ret)
	7003	+ return ret;
6442	7004
6443	7005	if (file->f_mode & FMODE_READ) {
6444	7006	iter = __tracing_open(inode, file, true);
..	..	@@ -6458,7 +7020,7 @@
6458	7020	ret = 0;
6459	7021
6460	7022	iter->tr = tr;
6461		- iter->trace_buffer = &tr->max_buffer;
	7023	+ iter->array_buffer = &tr->max_buffer;
6462	7024	iter->cpu_file = tracing_get_cpu(inode);
6463	7025	m->private = iter;
6464	7026	file->private_data = m;
..	..	@@ -6495,6 +7057,15 @@
6495	7057	goto out;
6496	7058	}
6497	7059
	7060	+ local_irq_disable();
	7061	+ arch_spin_lock(&tr->max_lock);
	7062	+ if (tr->cond_snapshot)
	7063	+ ret = -EBUSY;
	7064	+ arch_spin_unlock(&tr->max_lock);
	7065	+ local_irq_enable();
	7066	+ if (ret)
	7067	+ goto out;
	7068	+
6498	7069	switch (val) {
6499	7070	case 0:
6500	7071	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
..	..	@@ -6514,7 +7085,7 @@
6514	7085	#endif
6515	7086	if (tr->allocated_snapshot)
6516	7087	ret = resize_buffer_duplicate_size(&tr->max_buffer,
6517		- &tr->trace_buffer, iter->cpu_file);
	7088	+ &tr->array_buffer, iter->cpu_file);
6518	7089	else
6519	7090	ret = tracing_alloc_snapshot_instance(tr);
6520	7091	if (ret < 0)
..	..	@@ -6522,7 +7093,7 @@
6522	7093	local_irq_disable();
6523	7094	/* Now, we're going to swap */
6524	7095	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6525		- update_max_tr(tr, current, smp_processor_id());
	7096	+ update_max_tr(tr, current, smp_processor_id(), NULL);
6526	7097	else
6527	7098	update_max_tr_single(tr, current, iter->cpu_file);
6528	7099	local_irq_enable();
..	..	@@ -6532,7 +7103,7 @@
6532	7103	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6533	7104	tracing_reset_online_cpus(&tr->max_buffer);
6534	7105	else
6535		- tracing_reset(&tr->max_buffer, iter->cpu_file);
	7106	+ tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
6536	7107	}
6537	7108	break;
6538	7109	}
..	..	@@ -6576,6 +7147,7 @@
6576	7147	struct ftrace_buffer_info *info;
6577	7148	int ret;
6578	7149
	7150	+ /* The following checks for tracefs lockdown */
6579	7151	ret = tracing_buffers_open(inode, filp);
6580	7152	if (ret < 0)
6581	7153	return ret;
..	..	@@ -6588,7 +7160,7 @@
6588	7160	}
6589	7161
6590	7162	info->iter.snapshot = true;
6591		- info->iter.trace_buffer = &info->iter.tr->max_buffer;
	7163	+ info->iter.array_buffer = &info->iter.tr->max_buffer;
6592	7164
6593	7165	return ret;
6594	7166	}
..	..	@@ -6697,19 +7269,263 @@
6697	7269
6698	7270	#endif /* CONFIG_TRACER_SNAPSHOT */
6699	7271
	7272	+#define TRACING_LOG_ERRS_MAX 8
	7273	+#define TRACING_LOG_LOC_MAX 128
	7274	+
	7275	+#define CMD_PREFIX " Command: "
	7276	+
	7277	+struct err_info {
	7278	+ const char *errs; / ptr to loc-specific array of err strings */
	7279	+ u8 type; /* index into errs -> specific err string */
	7280	+ u8 pos; /* MAX_FILTER_STR_VAL = 256 */
	7281	+ u64 ts;
	7282	+};
	7283	+
	7284	+struct tracing_log_err {
	7285	+ struct list_head list;
	7286	+ struct err_info info;
	7287	+ char loc[TRACING_LOG_LOC_MAX]; /* err location */
	7288	+ char cmd[MAX_FILTER_STR_VAL]; /* what caused err */
	7289	+};
	7290	+
	7291	+static DEFINE_MUTEX(tracing_err_log_lock);
	7292	+
	7293	+static struct tracing_log_err get_tracing_log_err(struct trace_array tr)
	7294	+{
	7295	+ struct tracing_log_err *err;
	7296	+
	7297	+ if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
	7298	+ err = kzalloc(sizeof(*err), GFP_KERNEL);
	7299	+ if (!err)
	7300	+ err = ERR_PTR(-ENOMEM);
	7301	+ else
	7302	+ tr->n_err_log_entries++;
	7303	+
	7304	+ return err;
	7305	+ }
	7306	+
	7307	+ err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
	7308	+ list_del(&err->list);
	7309	+
	7310	+ return err;
	7311	+}
	7312	+
	7313	+/**
	7314	+ * err_pos - find the position of a string within a command for error careting
	7315	+ * @cmd: The tracing command that caused the error
	7316	+ * @str: The string to position the caret at within @cmd
	7317	+ *
	7318	+ * Finds the position of the first occurence of @str within @cmd. The
	7319	+ * return value can be passed to tracing_log_err() for caret placement
	7320	+ * within @cmd.
	7321	+ *
	7322	+ * Returns the index within @cmd of the first occurence of @str or 0
	7323	+ * if @str was not found.
	7324	+ */
	7325	+unsigned int err_pos(char cmd, const char str)
	7326	+{
	7327	+ char *found;
	7328	+
	7329	+ if (WARN_ON(!strlen(cmd)))
	7330	+ return 0;
	7331	+
	7332	+ found = strstr(cmd, str);
	7333	+ if (found)
	7334	+ return found - cmd;
	7335	+
	7336	+ return 0;
	7337	+}
	7338	+
	7339	+/**
	7340	+ * tracing_log_err - write an error to the tracing error log
	7341	+ * @tr: The associated trace array for the error (NULL for top level array)
	7342	+ * @loc: A string describing where the error occurred
	7343	+ * @cmd: The tracing command that caused the error
	7344	+ * @errs: The array of loc-specific static error strings
	7345	+ * @type: The index into errs[], which produces the specific static err string
	7346	+ * @pos: The position the caret should be placed in the cmd
	7347	+ *
	7348	+ * Writes an error into tracing/error_log of the form:
	7349	+ *
	7350	+ * <loc>: error: <text>
	7351	+ * Command: <cmd>
	7352	+ * ^
	7353	+ *
	7354	+ * tracing/error_log is a small log file containing the last
	7355	+ * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
	7356	+ * unless there has been a tracing error, and the error log can be
	7357	+ * cleared and have its memory freed by writing the empty string in
	7358	+ * truncation mode to it i.e. echo > tracing/error_log.
	7359	+ *
	7360	+ * NOTE: the @errs array along with the @type param are used to
	7361	+ * produce a static error string - this string is not copied and saved
	7362	+ * when the error is logged - only a pointer to it is saved. See
	7363	+ * existing callers for examples of how static strings are typically
	7364	+ * defined for use with tracing_log_err().
	7365	+ */
	7366	+void tracing_log_err(struct trace_array *tr,
	7367	+ const char loc, const char cmd,
	7368	+ const char **errs, u8 type, u8 pos)
	7369	+{
	7370	+ struct tracing_log_err *err;
	7371	+
	7372	+ if (!tr)
	7373	+ tr = &global_trace;
	7374	+
	7375	+ mutex_lock(&tracing_err_log_lock);
	7376	+ err = get_tracing_log_err(tr);
	7377	+ if (PTR_ERR(err) == -ENOMEM) {
	7378	+ mutex_unlock(&tracing_err_log_lock);
	7379	+ return;
	7380	+ }
	7381	+
	7382	+ snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
	7383	+ snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
	7384	+
	7385	+ err->info.errs = errs;
	7386	+ err->info.type = type;
	7387	+ err->info.pos = pos;
	7388	+ err->info.ts = local_clock();
	7389	+
	7390	+ list_add_tail(&err->list, &tr->err_log);
	7391	+ mutex_unlock(&tracing_err_log_lock);
	7392	+}
	7393	+
	7394	+static void clear_tracing_err_log(struct trace_array *tr)
	7395	+{
	7396	+ struct tracing_log_err err, next;
	7397	+
	7398	+ mutex_lock(&tracing_err_log_lock);
	7399	+ list_for_each_entry_safe(err, next, &tr->err_log, list) {
	7400	+ list_del(&err->list);
	7401	+ kfree(err);
	7402	+ }
	7403	+
	7404	+ tr->n_err_log_entries = 0;
	7405	+ mutex_unlock(&tracing_err_log_lock);
	7406	+}
	7407	+
	7408	+static void tracing_err_log_seq_start(struct seq_file m, loff_t *pos)
	7409	+{
	7410	+ struct trace_array *tr = m->private;
	7411	+
	7412	+ mutex_lock(&tracing_err_log_lock);
	7413	+
	7414	+ return seq_list_start(&tr->err_log, *pos);
	7415	+}
	7416	+
	7417	+static void tracing_err_log_seq_next(struct seq_file m, void v, loff_t pos)
	7418	+{
	7419	+ struct trace_array *tr = m->private;
	7420	+
	7421	+ return seq_list_next(v, &tr->err_log, pos);
	7422	+}
	7423	+
	7424	+static void tracing_err_log_seq_stop(struct seq_file m, void v)
	7425	+{
	7426	+ mutex_unlock(&tracing_err_log_lock);
	7427	+}
	7428	+
	7429	+static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
	7430	+{
	7431	+ u8 i;
	7432	+
	7433	+ for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
	7434	+ seq_putc(m, ' ');
	7435	+ for (i = 0; i < pos; i++)
	7436	+ seq_putc(m, ' ');
	7437	+ seq_puts(m, "^\n");
	7438	+}
	7439	+
	7440	+static int tracing_err_log_seq_show(struct seq_file m, void v)
	7441	+{
	7442	+ struct tracing_log_err *err = v;
	7443	+
	7444	+ if (err) {
	7445	+ const char *err_text = err->info.errs[err->info.type];
	7446	+ u64 sec = err->info.ts;
	7447	+ u32 nsec;
	7448	+
	7449	+ nsec = do_div(sec, NSEC_PER_SEC);
	7450	+ seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
	7451	+ err->loc, err_text);
	7452	+ seq_printf(m, "%s", err->cmd);
	7453	+ tracing_err_log_show_pos(m, err->info.pos);
	7454	+ }
	7455	+
	7456	+ return 0;
	7457	+}
	7458	+
	7459	+static const struct seq_operations tracing_err_log_seq_ops = {
	7460	+ .start = tracing_err_log_seq_start,
	7461	+ .next = tracing_err_log_seq_next,
	7462	+ .stop = tracing_err_log_seq_stop,
	7463	+ .show = tracing_err_log_seq_show
	7464	+};
	7465	+
	7466	+static int tracing_err_log_open(struct inode inode, struct file file)
	7467	+{
	7468	+ struct trace_array *tr = inode->i_private;
	7469	+ int ret = 0;
	7470	+
	7471	+ ret = tracing_check_open_get_tr(tr);
	7472	+ if (ret)
	7473	+ return ret;
	7474	+
	7475	+ /* If this file was opened for write, then erase contents */
	7476	+ if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
	7477	+ clear_tracing_err_log(tr);
	7478	+
	7479	+ if (file->f_mode & FMODE_READ) {
	7480	+ ret = seq_open(file, &tracing_err_log_seq_ops);
	7481	+ if (!ret) {
	7482	+ struct seq_file *m = file->private_data;
	7483	+ m->private = tr;
	7484	+ } else {
	7485	+ trace_array_put(tr);
	7486	+ }
	7487	+ }
	7488	+ return ret;
	7489	+}
	7490	+
	7491	+static ssize_t tracing_err_log_write(struct file *file,
	7492	+ const char __user *buffer,
	7493	+ size_t count, loff_t *ppos)
	7494	+{
	7495	+ return count;
	7496	+}
	7497	+
	7498	+static int tracing_err_log_release(struct inode inode, struct file file)
	7499	+{
	7500	+ struct trace_array *tr = inode->i_private;
	7501	+
	7502	+ trace_array_put(tr);
	7503	+
	7504	+ if (file->f_mode & FMODE_READ)
	7505	+ seq_release(inode, file);
	7506	+
	7507	+ return 0;
	7508	+}
	7509	+
	7510	+static const struct file_operations tracing_err_log_fops = {
	7511	+ .open = tracing_err_log_open,
	7512	+ .write = tracing_err_log_write,
	7513	+ .read = seq_read,
	7514	+ .llseek = seq_lseek,
	7515	+ .release = tracing_err_log_release,
	7516	+};
	7517	+
6700	7518	static int tracing_buffers_open(struct inode inode, struct file filp)
6701	7519	{
6702	7520	struct trace_array *tr = inode->i_private;
6703	7521	struct ftrace_buffer_info *info;
6704	7522	int ret;
6705	7523
6706		- if (tracing_disabled)
6707		- return -ENODEV;
	7524	+ ret = tracing_check_open_get_tr(tr);
	7525	+ if (ret)
	7526	+ return ret;
6708	7527
6709		- if (trace_array_get(tr) < 0)
6710		- return -ENODEV;
6711		-
6712		- info = kzalloc(sizeof(*info), GFP_KERNEL);
	7528	+ info = kvzalloc(sizeof(*info), GFP_KERNEL);
6713	7529	if (!info) {
6714	7530	trace_array_put(tr);
6715	7531	return -ENOMEM;
..	..	@@ -6720,14 +7536,14 @@
6720	7536	info->iter.tr = tr;
6721	7537	info->iter.cpu_file = tracing_get_cpu(inode);
6722	7538	info->iter.trace = tr->current_trace;
6723		- info->iter.trace_buffer = &tr->trace_buffer;
	7539	+ info->iter.array_buffer = &tr->array_buffer;
6724	7540	info->spare = NULL;
6725	7541	/* Force reading ring buffer for first read */
6726	7542	info->read = (unsigned int)-1;
6727	7543
6728	7544	filp->private_data = info;
6729	7545
6730		- tr->current_trace->ref++;
	7546	+ tr->trace_ref++;
6731	7547
6732	7548	mutex_unlock(&trace_types_lock);
6733	7549
..	..	@@ -6765,7 +7581,7 @@
6765	7581	#endif
6766	7582
6767	7583	if (!info->spare) {
6768		- info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
	7584	+ info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
6769	7585	iter->cpu_file);
6770	7586	if (IS_ERR(info->spare)) {
6771	7587	ret = PTR_ERR(info->spare);
..	..	@@ -6783,7 +7599,7 @@
6783	7599
6784	7600	again:
6785	7601	trace_access_lock(iter->cpu_file);
6786		- ret = ring_buffer_read_page(iter->trace_buffer->buffer,
	7602	+ ret = ring_buffer_read_page(iter->array_buffer->buffer,
6787	7603	&info->spare,
6788	7604	count,
6789	7605	iter->cpu_file, 0);
..	..	@@ -6794,7 +7610,7 @@
6794	7610	if ((filp->f_flags & O_NONBLOCK))
6795	7611	return -EAGAIN;
6796	7612
6797		- ret = wait_on_pipe(iter, false);
	7613	+ ret = wait_on_pipe(iter, 0);
6798	7614	if (ret)
6799	7615	return ret;
6800	7616
..	..	@@ -6828,14 +7644,14 @@
6828	7644
6829	7645	mutex_lock(&trace_types_lock);
6830	7646
6831		- iter->tr->current_trace->ref--;
	7647	+ iter->tr->trace_ref--;
6832	7648
6833	7649	__trace_array_put(iter->tr);
6834	7650
6835	7651	if (info->spare)
6836		- ring_buffer_free_read_page(iter->trace_buffer->buffer,
	7652	+ ring_buffer_free_read_page(iter->array_buffer->buffer,
6837	7653	info->spare_cpu, info->spare);
6838		- kfree(info);
	7654	+ kvfree(info);
6839	7655
6840	7656	mutex_unlock(&trace_types_lock);
6841	7657
..	..	@@ -6843,7 +7659,7 @@
6843	7659	}
6844	7660
6845	7661	struct buffer_ref {
6846		- struct ring_buffer *buffer;
	7662	+ struct trace_buffer *buffer;
6847	7663	void *page;
6848	7664	int cpu;
6849	7665	refcount_t refcount;
..	..	@@ -6880,10 +7696,7 @@
6880	7696
6881	7697	/* Pipe buffer operations for a buffer. */
6882	7698	static const struct pipe_buf_operations buffer_pipe_buf_ops = {
6883		- .can_merge = 0,
6884		- .confirm = generic_pipe_buf_confirm,
6885	7699	.release = buffer_pipe_buf_release,
6886		- .steal = generic_pipe_buf_nosteal,
6887	7700	.get = buffer_pipe_buf_get,
6888	7701	};
6889	7702
..	..	@@ -6939,7 +7752,7 @@
6939	7752
6940	7753	again:
6941	7754	trace_access_lock(iter->cpu_file);
6942		- entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
	7755	+ entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
6943	7756
6944	7757	for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
6945	7758	struct page *page;
..	..	@@ -6952,7 +7765,7 @@
6952	7765	}
6953	7766
6954	7767	refcount_set(&ref->refcount, 1);
6955		- ref->buffer = iter->trace_buffer->buffer;
	7768	+ ref->buffer = iter->array_buffer->buffer;
6956	7769	ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
6957	7770	if (IS_ERR(ref->page)) {
6958	7771	ret = PTR_ERR(ref->page);
..	..	@@ -6980,7 +7793,7 @@
6980	7793	spd.nr_pages++;
6981	7794	*ppos += PAGE_SIZE;
6982	7795
6983		- entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
	7796	+ entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
6984	7797	}
6985	7798
6986	7799	trace_access_unlock(iter->cpu_file);
..	..	@@ -6995,7 +7808,7 @@
6995	7808	if ((file->f_flags & O_NONBLOCK) \|\| (flags & SPLICE_F_NONBLOCK))
6996	7809	goto out;
6997	7810
6998		- ret = wait_on_pipe(iter, true);
	7811	+ ret = wait_on_pipe(iter, iter->tr->buffer_percent);
6999	7812	if (ret)
7000	7813	goto out;
7001	7814
..	..	@@ -7024,7 +7837,7 @@
7024	7837	{
7025	7838	struct inode *inode = file_inode(filp);
7026	7839	struct trace_array *tr = inode->i_private;
7027		- struct trace_buffer *trace_buf = &tr->trace_buffer;
	7840	+ struct array_buffer *trace_buf = &tr->array_buffer;
7028	7841	int cpu = tracing_get_cpu(inode);
7029	7842	struct trace_seq *s;
7030	7843	unsigned long cnt;
..	..	@@ -7095,14 +7908,23 @@
7095	7908	tracing_read_dyn_info(struct file filp, char __user ubuf,
7096	7909	size_t cnt, loff_t *ppos)
7097	7910	{
7098		- unsigned long *p = filp->private_data;
7099		- char buf[64]; /* Not too big for a shallow stack */
	7911	+ ssize_t ret;
	7912	+ char *buf;
7100	7913	int r;
7101	7914
7102		- r = scnprintf(buf, 63, "%ld", *p);
7103		- buf[r++] = '\n';
	7915	+ /* 256 should be plenty to hold the amount needed */
	7916	+ buf = kmalloc(256, GFP_KERNEL);
	7917	+ if (!buf)
	7918	+ return -ENOMEM;
7104	7919
7105		- return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
	7920	+ r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
	7921	+ ftrace_update_tot_cnt,
	7922	+ ftrace_number_of_pages,
	7923	+ ftrace_number_of_groups);
	7924	+
	7925	+ ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
	7926	+ kfree(buf);
	7927	+ return ret;
7106	7928	}
7107	7929
7108	7930	static const struct file_operations tracing_dyn_info_fops = {
..	..	@@ -7296,7 +8118,7 @@
7296	8118
7297	8119	tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
7298	8120
7299		- WARN_ONCE(!tr->percpu_dir,
	8121	+ MEM_FAIL(!tr->percpu_dir,
7300	8122	"Could not create tracefs directory 'per_cpu/%d'\n", cpu);
7301	8123
7302	8124	return tr->percpu_dir;
..	..	@@ -7617,7 +8439,7 @@
7617	8439	for (cnt = 0; opts[cnt].name; cnt++) {
7618	8440	create_trace_option_file(tr, &topts[cnt], flags,
7619	8441	&opts[cnt]);
7620		- WARN_ONCE(topts[cnt].entry == NULL,
	8442	+ MEM_FAIL(topts[cnt].entry == NULL,
7621	8443	"Failed to create trace option: %s",
7622	8444	opts[cnt].name);
7623	8445	}
..	..	@@ -7674,7 +8496,7 @@
7674	8496	size_t cnt, loff_t *ppos)
7675	8497	{
7676	8498	struct trace_array *tr = filp->private_data;
7677		- struct ring_buffer *buffer = tr->trace_buffer.buffer;
	8499	+ struct trace_buffer *buffer = tr->array_buffer.buffer;
7678	8500	unsigned long val;
7679	8501	int ret;
7680	8502
..	..	@@ -7711,13 +8533,60 @@
7711	8533	.llseek = default_llseek,
7712	8534	};
7713	8535
7714		-struct dentry *trace_instance_dir;
	8536	+static ssize_t
	8537	+buffer_percent_read(struct file filp, char __user ubuf,
	8538	+ size_t cnt, loff_t *ppos)
	8539	+{
	8540	+ struct trace_array *tr = filp->private_data;
	8541	+ char buf[64];
	8542	+ int r;
	8543	+
	8544	+ r = tr->buffer_percent;
	8545	+ r = sprintf(buf, "%d\n", r);
	8546	+
	8547	+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
	8548	+}
	8549	+
	8550	+static ssize_t
	8551	+buffer_percent_write(struct file filp, const char __user ubuf,
	8552	+ size_t cnt, loff_t *ppos)
	8553	+{
	8554	+ struct trace_array *tr = filp->private_data;
	8555	+ unsigned long val;
	8556	+ int ret;
	8557	+
	8558	+ ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
	8559	+ if (ret)
	8560	+ return ret;
	8561	+
	8562	+ if (val > 100)
	8563	+ return -EINVAL;
	8564	+
	8565	+ if (!val)
	8566	+ val = 1;
	8567	+
	8568	+ tr->buffer_percent = val;
	8569	+
	8570	+ (*ppos)++;
	8571	+
	8572	+ return cnt;
	8573	+}
	8574	+
	8575	+static const struct file_operations buffer_percent_fops = {
	8576	+ .open = tracing_open_generic_tr,
	8577	+ .read = buffer_percent_read,
	8578	+ .write = buffer_percent_write,
	8579	+ .release = tracing_release_generic_tr,
	8580	+ .llseek = default_llseek,
	8581	+};
	8582	+
	8583	+static struct dentry *trace_instance_dir;
7715	8584
7716	8585	static void
7717	8586	init_tracer_tracefs(struct trace_array tr, struct dentry d_tracer);
7718	8587
7719	8588	static int
7720		-allocate_trace_buffer(struct trace_array tr, struct trace_buffer buf, int size)
	8589	+allocate_trace_buffer(struct trace_array tr, struct array_buffer buf, int size)
7721	8590	{
7722	8591	enum ring_buffer_flags rb_flags;
7723	8592
..	..	@@ -7737,8 +8606,8 @@
7737	8606	}
7738	8607
7739	8608	/* Allocate the first page for all buffers */
7740		- set_buffer_entries(&tr->trace_buffer,
7741		- ring_buffer_size(tr->trace_buffer.buffer, 0));
	8609	+ set_buffer_entries(&tr->array_buffer,
	8610	+ ring_buffer_size(tr->array_buffer.buffer, 0));
7742	8611
7743	8612	return 0;
7744	8613	}
..	..	@@ -7747,18 +8616,18 @@
7747	8616	{
7748	8617	int ret;
7749	8618
7750		- ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
	8619	+ ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
7751	8620	if (ret)
7752	8621	return ret;
7753	8622
7754	8623	#ifdef CONFIG_TRACER_MAX_TRACE
7755	8624	ret = allocate_trace_buffer(tr, &tr->max_buffer,
7756	8625	allocate_snapshot ? size : 1);
7757		- if (WARN_ON(ret)) {
7758		- ring_buffer_free(tr->trace_buffer.buffer);
7759		- tr->trace_buffer.buffer = NULL;
7760		- free_percpu(tr->trace_buffer.data);
7761		- tr->trace_buffer.data = NULL;
	8626	+ if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
	8627	+ ring_buffer_free(tr->array_buffer.buffer);
	8628	+ tr->array_buffer.buffer = NULL;
	8629	+ free_percpu(tr->array_buffer.data);
	8630	+ tr->array_buffer.data = NULL;
7762	8631	return -ENOMEM;
7763	8632	}
7764	8633	tr->allocated_snapshot = allocate_snapshot;
..	..	@@ -7770,22 +8639,10 @@
7770	8639	allocate_snapshot = false;
7771	8640	#endif
7772	8641
7773		- /*
7774		- * Because of some magic with the way alloc_percpu() works on
7775		- * x86_64, we need to synchronize the pgd of all the tables,
7776		- * otherwise the trace events that happen in x86_64 page fault
7777		- * handlers can't cope with accessing the chance that a
7778		- * alloc_percpu()'d memory might be touched in the page fault trace
7779		- * event. Oh, and we need to audit all other alloc_percpu() and vmalloc()
7780		- * calls in tracing, because something might get triggered within a
7781		- * page fault trace event!
7782		- */
7783		- vmalloc_sync_mappings();
7784		-
7785	8642	return 0;
7786	8643	}
7787	8644
7788		-static void free_trace_buffer(struct trace_buffer *buf)
	8645	+static void free_trace_buffer(struct array_buffer *buf)
7789	8646	{
7790	8647	if (buf->buffer) {
7791	8648	ring_buffer_free(buf->buffer);
..	..	@@ -7800,7 +8657,7 @@
7800	8657	if (!tr)
7801	8658	return;
7802	8659
7803		- free_trace_buffer(&tr->trace_buffer);
	8660	+ free_trace_buffer(&tr->array_buffer);
7804	8661
7805	8662	#ifdef CONFIG_TRACER_MAX_TRACE
7806	8663	free_trace_buffer(&tr->max_buffer);
..	..	@@ -7827,28 +8684,68 @@
7827	8684	static void update_tracer_options(struct trace_array *tr)
7828	8685	{
7829	8686	mutex_lock(&trace_types_lock);
	8687	+ tracer_options_updated = true;
7830	8688	__update_tracer_options(tr);
7831	8689	mutex_unlock(&trace_types_lock);
7832	8690	}
7833	8691
7834		-static int instance_mkdir(const char *name)
	8692	+/* Must have trace_types_lock held */
	8693	+struct trace_array trace_array_find(const char instance)
	8694	+{
	8695	+ struct trace_array tr, found = NULL;
	8696	+
	8697	+ list_for_each_entry(tr, &ftrace_trace_arrays, list) {
	8698	+ if (tr->name && strcmp(tr->name, instance) == 0) {
	8699	+ found = tr;
	8700	+ break;
	8701	+ }
	8702	+ }
	8703	+
	8704	+ return found;
	8705	+}
	8706	+
	8707	+struct trace_array trace_array_find_get(const char instance)
	8708	+{
	8709	+ struct trace_array *tr;
	8710	+
	8711	+ mutex_lock(&trace_types_lock);
	8712	+ tr = trace_array_find(instance);
	8713	+ if (tr)
	8714	+ tr->ref++;
	8715	+ mutex_unlock(&trace_types_lock);
	8716	+
	8717	+ return tr;
	8718	+}
	8719	+
	8720	+static int trace_array_create_dir(struct trace_array *tr)
	8721	+{
	8722	+ int ret;
	8723	+
	8724	+ tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);
	8725	+ if (!tr->dir)
	8726	+ return -EINVAL;
	8727	+
	8728	+ ret = event_trace_add_tracer(tr->dir, tr);
	8729	+ if (ret) {
	8730	+ tracefs_remove(tr->dir);
	8731	+ return ret;
	8732	+ }
	8733	+
	8734	+ init_tracer_tracefs(tr, tr->dir);
	8735	+ __update_tracer_options(tr);
	8736	+
	8737	+ return ret;
	8738	+}
	8739	+
	8740	+static struct trace_array trace_array_create(const char name)
7835	8741	{
7836	8742	struct trace_array *tr;
7837	8743	int ret;
7838	8744
7839		- mutex_lock(&event_mutex);
7840		- mutex_lock(&trace_types_lock);
7841		-
7842		- ret = -EEXIST;
7843		- list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7844		- if (tr->name && strcmp(tr->name, name) == 0)
7845		- goto out_unlock;
7846		- }
7847		-
7848	8745	ret = -ENOMEM;
7849	8746	tr = kzalloc(sizeof(*tr), GFP_KERNEL);
7850	8747	if (!tr)
7851		- goto out_unlock;
	8748	+ return ERR_PTR(ret);
7852	8749
7853	8750	tr->name = kstrdup(name, GFP_KERNEL);
7854	8751	if (!tr->name)
..	..	@@ -7870,70 +8767,112 @@
7870	8767	INIT_LIST_HEAD(&tr->systems);
7871	8768	INIT_LIST_HEAD(&tr->events);
7872	8769	INIT_LIST_HEAD(&tr->hist_vars);
	8770	+ INIT_LIST_HEAD(&tr->err_log);
7873	8771
7874	8772	if (allocate_trace_buffers(tr, trace_buf_size) < 0)
7875	8773	goto out_free_tr;
7876	8774
7877		- tr->dir = tracefs_create_dir(name, trace_instance_dir);
7878		- if (!tr->dir)
	8775	+ if (ftrace_allocate_ftrace_ops(tr) < 0)
7879	8776	goto out_free_tr;
7880		-
7881		- ret = event_trace_add_tracer(tr->dir, tr);
7882		- if (ret) {
7883		- tracefs_remove_recursive(tr->dir);
7884		- goto out_free_tr;
7885		- }
7886	8777
7887	8778	ftrace_init_trace_array(tr);
7888	8779
7889		- init_tracer_tracefs(tr, tr->dir);
7890	8780	init_trace_flags_index(tr);
7891		- __update_tracer_options(tr);
	8781	+
	8782	+ if (trace_instance_dir) {
	8783	+ ret = trace_array_create_dir(tr);
	8784	+ if (ret)
	8785	+ goto out_free_tr;
	8786	+ } else
	8787	+ __trace_early_add_events(tr);
7892	8788
7893	8789	list_add(&tr->list, &ftrace_trace_arrays);
7894	8790
7895		- mutex_unlock(&trace_types_lock);
7896		- mutex_unlock(&event_mutex);
	8791	+ tr->ref++;
7897	8792
7898		- return 0;
	8793	+ return tr;
7899	8794
7900	8795	out_free_tr:
	8796	+ ftrace_free_ftrace_ops(tr);
7901	8797	free_trace_buffers(tr);
7902	8798	free_cpumask_var(tr->tracing_cpumask);
7903	8799	kfree(tr->name);
7904	8800	kfree(tr);
7905	8801
7906		- out_unlock:
7907		- mutex_unlock(&trace_types_lock);
7908		- mutex_unlock(&event_mutex);
7909		-
7910		- return ret;
7911		-
	8802	+ return ERR_PTR(ret);
7912	8803	}
7913	8804
7914		-static int instance_rmdir(const char *name)
	8805	+static int instance_mkdir(const char *name)
7915	8806	{
7916	8807	struct trace_array *tr;
7917		- int found = 0;
7918	8808	int ret;
7919		- int i;
7920	8809
7921	8810	mutex_lock(&event_mutex);
7922	8811	mutex_lock(&trace_types_lock);
7923	8812
7924		- ret = -ENODEV;
7925		- list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7926		- if (tr->name && strcmp(tr->name, name) == 0) {
7927		- found = 1;
7928		- break;
7929		- }
7930		- }
7931		- if (!found)
	8813	+ ret = -EEXIST;
	8814	+ if (trace_array_find(name))
7932	8815	goto out_unlock;
7933	8816
7934		- ret = -EBUSY;
7935		- if (tr->ref \|\| (tr->current_trace && tr->current_trace->ref))
7936		- goto out_unlock;
	8817	+ tr = trace_array_create(name);
	8818	+
	8819	+ ret = PTR_ERR_OR_ZERO(tr);
	8820	+
	8821	+out_unlock:
	8822	+ mutex_unlock(&trace_types_lock);
	8823	+ mutex_unlock(&event_mutex);
	8824	+ return ret;
	8825	+}
	8826	+
	8827	+/**
	8828	+ * trace_array_get_by_name - Create/Lookup a trace array, given its name.
	8829	+ * @name: The name of the trace array to be looked up/created.
	8830	+ *
	8831	+ * Returns pointer to trace array with given name.
	8832	+ * NULL, if it cannot be created.
	8833	+ *
	8834	+ * NOTE: This function increments the reference counter associated with the
	8835	+ * trace array returned. This makes sure it cannot be freed while in use.
	8836	+ * Use trace_array_put() once the trace array is no longer needed.
	8837	+ * If the trace_array is to be freed, trace_array_destroy() needs to
	8838	+ * be called after the trace_array_put(), or simply let user space delete
	8839	+ * it from the tracefs instances directory. But until the
	8840	+ * trace_array_put() is called, user space can not delete it.
	8841	+ *
	8842	+ */
	8843	+struct trace_array trace_array_get_by_name(const char name)
	8844	+{
	8845	+ struct trace_array *tr;
	8846	+
	8847	+ mutex_lock(&event_mutex);
	8848	+ mutex_lock(&trace_types_lock);
	8849	+
	8850	+ list_for_each_entry(tr, &ftrace_trace_arrays, list) {
	8851	+ if (tr->name && strcmp(tr->name, name) == 0)
	8852	+ goto out_unlock;
	8853	+ }
	8854	+
	8855	+ tr = trace_array_create(name);
	8856	+
	8857	+ if (IS_ERR(tr))
	8858	+ tr = NULL;
	8859	+out_unlock:
	8860	+ if (tr)
	8861	+ tr->ref++;
	8862	+
	8863	+ mutex_unlock(&trace_types_lock);
	8864	+ mutex_unlock(&event_mutex);
	8865	+ return tr;
	8866	+}
	8867	+EXPORT_SYMBOL_GPL(trace_array_get_by_name);
	8868	+
	8869	+static int __remove_instance(struct trace_array *tr)
	8870	+{
	8871	+ int i;
	8872	+
	8873	+ /* Reference counter for a newly created trace array = 1. */
	8874	+ if (tr->ref > 1 \|\| (tr->current_trace && tr->trace_ref))
	8875	+ return -EBUSY;
7937	8876
7938	8877	list_del(&tr->list);
7939	8878
..	..	@@ -7948,7 +8887,7 @@
7948	8887	event_trace_del_tracer(tr);
7949	8888	ftrace_clear_pids(tr);
7950	8889	ftrace_destroy_function_files(tr);
7951		- tracefs_remove_recursive(tr->dir);
	8890	+ tracefs_remove(tr->dir);
7952	8891	free_trace_buffers(tr);
7953	8892
7954	8893	for (i = 0; i < tr->nr_topts; i++) {
..	..	@@ -7960,9 +8899,50 @@
7960	8899	kfree(tr->name);
7961	8900	kfree(tr);
7962	8901
7963		- ret = 0;
	8902	+ return 0;
	8903	+}
7964	8904
7965		- out_unlock:
	8905	+int trace_array_destroy(struct trace_array *this_tr)
	8906	+{
	8907	+ struct trace_array *tr;
	8908	+ int ret;
	8909	+
	8910	+ if (!this_tr)
	8911	+ return -EINVAL;
	8912	+
	8913	+ mutex_lock(&event_mutex);
	8914	+ mutex_lock(&trace_types_lock);
	8915	+
	8916	+ ret = -ENODEV;
	8917	+
	8918	+ /* Making sure trace array exists before destroying it. */
	8919	+ list_for_each_entry(tr, &ftrace_trace_arrays, list) {
	8920	+ if (tr == this_tr) {
	8921	+ ret = __remove_instance(tr);
	8922	+ break;
	8923	+ }
	8924	+ }
	8925	+
	8926	+ mutex_unlock(&trace_types_lock);
	8927	+ mutex_unlock(&event_mutex);
	8928	+
	8929	+ return ret;
	8930	+}
	8931	+EXPORT_SYMBOL_GPL(trace_array_destroy);
	8932	+
	8933	+static int instance_rmdir(const char *name)
	8934	+{
	8935	+ struct trace_array *tr;
	8936	+ int ret;
	8937	+
	8938	+ mutex_lock(&event_mutex);
	8939	+ mutex_lock(&trace_types_lock);
	8940	+
	8941	+ ret = -ENODEV;
	8942	+ tr = trace_array_find(name);
	8943	+ if (tr)
	8944	+ ret = __remove_instance(tr);
	8945	+
7966	8946	mutex_unlock(&trace_types_lock);
7967	8947	mutex_unlock(&event_mutex);
7968	8948
..	..	@@ -7971,11 +8951,27 @@
7971	8951
7972	8952	static __init void create_trace_instances(struct dentry *d_tracer)
7973	8953	{
	8954	+ struct trace_array *tr;
	8955	+
7974	8956	trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
7975	8957	instance_mkdir,
7976	8958	instance_rmdir);
7977		- if (WARN_ON(!trace_instance_dir))
	8959	+ if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
7978	8960	return;
	8961	+
	8962	+ mutex_lock(&event_mutex);
	8963	+ mutex_lock(&trace_types_lock);
	8964	+
	8965	+ list_for_each_entry(tr, &ftrace_trace_arrays, list) {
	8966	+ if (!tr->name)
	8967	+ continue;
	8968	+ if (MEM_FAIL(trace_array_create_dir(tr) < 0,
	8969	+ "Failed to create instance directory\n"))
	8970	+ break;
	8971	+ }
	8972	+
	8973	+ mutex_unlock(&trace_types_lock);
	8974	+ mutex_unlock(&event_mutex);
7979	8975	}
7980	8976
7981	8977	static void
..	..	@@ -8032,20 +9028,27 @@
8032	9028	trace_create_file("timestamp_mode", 0444, d_tracer, tr,
8033	9029	&trace_time_stamp_mode_fops);
8034	9030
	9031	+ tr->buffer_percent = 50;
	9032	+
	9033	+ trace_create_file("buffer_percent", 0444, d_tracer,
	9034	+ tr, &buffer_percent_fops);
	9035	+
8035	9036	create_trace_options_dir(tr);
8036	9037
8037	9038	#if defined(CONFIG_TRACER_MAX_TRACE) \|\| defined(CONFIG_HWLAT_TRACER)
8038		- trace_create_file("tracing_max_latency", 0644, d_tracer,
8039		- &tr->max_latency, &tracing_max_lat_fops);
	9039	+ trace_create_maxlat_file(tr, d_tracer);
8040	9040	#endif
8041	9041
8042	9042	if (ftrace_create_function_files(tr, d_tracer))
8043		- WARN(1, "Could not allocate function filter files");
	9043	+ MEM_FAIL(1, "Could not allocate function filter files");
8044	9044
8045	9045	#ifdef CONFIG_TRACER_SNAPSHOT
8046	9046	trace_create_file("snapshot", 0644, d_tracer,
8047	9047	tr, &snapshot_fops);
8048	9048	#endif
	9049	+
	9050	+ trace_create_file("error_log", 0644, d_tracer,
	9051	+ tr, &tracing_err_log_fops);
8049	9052
8050	9053	for_each_tracing_cpu(cpu)
8051	9054	tracing_init_tracefs_percpu(tr, cpu);
..	..	@@ -8053,6 +9056,7 @@
8053	9056	ftrace_init_tracefs(tr, d_tracer);
8054	9057	}
8055	9058
	9059	+#ifndef CONFIG_TRACEFS_DISABLE_AUTOMOUNT
8056	9060	static struct vfsmount trace_automount(struct dentry mntpt, void *ingore)
8057	9061	{
8058	9062	struct vfsmount *mnt;
..	..	@@ -8074,6 +9078,7 @@
8074	9078
8075	9079	return mnt;
8076	9080	}
	9081	+#endif
8077	9082
8078	9083	/**
8079	9084	* tracing_init_dentry - initialize top level trace array
..	..	@@ -8082,19 +9087,23 @@
8082	9087	* directory. It is called via fs_initcall() by any of the boot up code
8083	9088	* and expects to return the dentry of the top level tracing directory.
8084	9089	*/
8085		-struct dentry *tracing_init_dentry(void)
	9090	+int tracing_init_dentry(void)
8086	9091	{
8087	9092	struct trace_array *tr = &global_trace;
8088	9093
	9094	+ if (security_locked_down(LOCKDOWN_TRACEFS)) {
	9095	+ pr_warn("Tracing disabled due to lockdown\n");
	9096	+ return -EPERM;
	9097	+ }
	9098	+
8089	9099	/* The top level trace array uses NULL as parent */
8090	9100	if (tr->dir)
8091		- return NULL;
	9101	+ return 0;
8092	9102
8093		- if (WARN_ON(!tracefs_initialized()) \|\|
8094		- (IS_ENABLED(CONFIG_DEBUG_FS) &&
8095		- WARN_ON(!debugfs_initialized())))
8096		- return ERR_PTR(-ENODEV);
	9103	+ if (WARN_ON(!tracefs_initialized()))
	9104	+ return -ENODEV;
8097	9105
	9106	+#ifndef CONFIG_TRACEFS_DISABLE_AUTOMOUNT
8098	9107	/*
8099	9108	* As there may still be users that expect the tracing
8100	9109	* files to exist in debugfs/tracing, we must automount
..	..	@@ -8103,12 +9112,11 @@
8103	9112	*/
8104	9113	tr->dir = debugfs_create_automount("tracing", NULL,
8105	9114	trace_automount, NULL);
8106		- if (!tr->dir) {
8107		- pr_warn_once("Could not create debugfs directory 'tracing'\n");
8108		- return ERR_PTR(-ENOMEM);
8109		- }
	9115	+#else
	9116	+ tr->dir = ERR_PTR(-ENODEV);
	9117	+#endif
8110	9118
8111		- return NULL;
	9119	+ return 0;
8112	9120	}
8113	9121
8114	9122	extern struct trace_eval_map *__start_ftrace_eval_maps[];
..	..	@@ -8184,7 +9192,7 @@
8184	9192	break;
8185	9193	}
8186	9194
8187		- return 0;
	9195	+ return NOTIFY_OK;
8188	9196	}
8189	9197
8190	9198	static struct notifier_block trace_module_nb = {
..	..	@@ -8195,48 +9203,48 @@
8195	9203
8196	9204	static __init int tracer_init_tracefs(void)
8197	9205	{
8198		- struct dentry *d_tracer;
	9206	+ int ret;
8199	9207
8200	9208	trace_access_lock_init();
8201	9209
8202		- d_tracer = tracing_init_dentry();
8203		- if (IS_ERR(d_tracer))
	9210	+ ret = tracing_init_dentry();
	9211	+ if (ret)
8204	9212	return 0;
8205	9213
8206	9214	event_trace_init();
8207	9215
8208		- init_tracer_tracefs(&global_trace, d_tracer);
8209		- ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
	9216	+ init_tracer_tracefs(&global_trace, NULL);
	9217	+ ftrace_init_tracefs_toplevel(&global_trace, NULL);
8210	9218
8211		- trace_create_file("tracing_thresh", 0644, d_tracer,
	9219	+ trace_create_file("tracing_thresh", 0644, NULL,
8212	9220	&global_trace, &tracing_thresh_fops);
8213	9221
8214		- trace_create_file("README", 0444, d_tracer,
	9222	+ trace_create_file("README", 0444, NULL,
8215	9223	NULL, &tracing_readme_fops);
8216	9224
8217		- trace_create_file("saved_cmdlines", 0444, d_tracer,
	9225	+ trace_create_file("saved_cmdlines", 0444, NULL,
8218	9226	NULL, &tracing_saved_cmdlines_fops);
8219	9227
8220		- trace_create_file("saved_cmdlines_size", 0644, d_tracer,
	9228	+ trace_create_file("saved_cmdlines_size", 0644, NULL,
8221	9229	NULL, &tracing_saved_cmdlines_size_fops);
8222	9230
8223		- trace_create_file("saved_tgids", 0444, d_tracer,
	9231	+ trace_create_file("saved_tgids", 0444, NULL,
8224	9232	NULL, &tracing_saved_tgids_fops);
8225	9233
8226	9234	trace_eval_init();
8227	9235
8228		- trace_create_eval_file(d_tracer);
	9236	+ trace_create_eval_file(NULL);
8229	9237
8230	9238	#ifdef CONFIG_MODULES
8231	9239	register_module_notifier(&trace_module_nb);
8232	9240	#endif
8233	9241
8234	9242	#ifdef CONFIG_DYNAMIC_FTRACE
8235		- trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
8236		- &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
	9243	+ trace_create_file("dyn_ftrace_total_info", 0444, NULL,
	9244	+ NULL, &tracing_dyn_info_fops);
8237	9245	#endif
8238	9246
8239		- create_trace_instances(d_tracer);
	9247	+ create_trace_instances(NULL);
8240	9248
8241	9249	update_tracer_options(&global_trace);
8242	9250
..	..	@@ -8246,8 +9254,17 @@
8246	9254	static int trace_panic_handler(struct notifier_block *this,
8247	9255	unsigned long event, void *unused)
8248	9256	{
	9257	+ bool ftrace_check = false;
	9258	+
	9259	+ trace_android_vh_ftrace_oops_enter(&ftrace_check);
	9260	+
	9261	+ if (ftrace_check)
	9262	+ return NOTIFY_OK;
	9263	+
8249	9264	if (ftrace_dump_on_oops)
8250	9265	ftrace_dump(ftrace_dump_on_oops);
	9266	+
	9267	+ trace_android_vh_ftrace_oops_exit(&ftrace_check);
8251	9268	return NOTIFY_OK;
8252	9269	}
8253	9270
..	..	@@ -8261,6 +9278,13 @@
8261	9278	unsigned long val,
8262	9279	void *data)
8263	9280	{
	9281	+ bool ftrace_check = false;
	9282	+
	9283	+ trace_android_vh_ftrace_oops_enter(&ftrace_check);
	9284	+
	9285	+ if (ftrace_check)
	9286	+ return NOTIFY_OK;
	9287	+
8264	9288	switch (val) {
8265	9289	case DIE_OOPS:
8266	9290	if (ftrace_dump_on_oops)
..	..	@@ -8269,6 +9293,8 @@
8269	9293	default:
8270	9294	break;
8271	9295	}
	9296	+
	9297	+ trace_android_vh_ftrace_oops_exit(&ftrace_check);
8272	9298	return NOTIFY_OK;
8273	9299	}
8274	9300
..	..	@@ -8293,6 +9319,8 @@
8293	9319	void
8294	9320	trace_printk_seq(struct trace_seq *s)
8295	9321	{
	9322	+ bool dump_printk = true;
	9323	+
8296	9324	/* Probably should print a warning here. */
8297	9325	if (s->seq.len >= TRACE_MAX_PRINT)
8298	9326	s->seq.len = TRACE_MAX_PRINT;
..	..	@@ -8308,7 +9336,9 @@
8308	9336	/* should be zero ended, but we are paranoid. */
8309	9337	s->buffer[s->seq.len] = 0;
8310	9338
8311		- printk(KERN_TRACE "%s", s->buffer);
	9339	+ trace_android_vh_ftrace_dump_buffer(s, &dump_printk);
	9340	+ if (dump_printk)
	9341	+ printk(KERN_TRACE "%s", s->buffer);
8312	9342
8313	9343	trace_seq_init(s);
8314	9344	}
..	..	@@ -8318,13 +9348,13 @@
8318	9348	iter->tr = &global_trace;
8319	9349	iter->trace = iter->tr->current_trace;
8320	9350	iter->cpu_file = RING_BUFFER_ALL_CPUS;
8321		- iter->trace_buffer = &global_trace.trace_buffer;
	9351	+ iter->array_buffer = &global_trace.array_buffer;
8322	9352
8323	9353	if (iter->trace && iter->trace->open)
8324	9354	iter->trace->open(iter);
8325	9355
8326	9356	/* Annotate start of buffers if we had overruns */
8327		- if (ring_buffer_overruns(iter->trace_buffer->buffer))
	9357	+ if (ring_buffer_overruns(iter->array_buffer->buffer))
8328	9358	iter->iter_flags \|= TRACE_FILE_ANNOTATE;
8329	9359
8330	9360	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
..	..	@@ -8341,6 +9371,8 @@
8341	9371	unsigned int old_userobj;
8342	9372	unsigned long flags;
8343	9373	int cnt = 0, cpu;
	9374	+ bool ftrace_check = false;
	9375	+ unsigned long size;
8344	9376
8345	9377	/* Only allow one dump user at a time. */
8346	9378	if (atomic_inc_return(&dump_running) != 1) {
..	..	@@ -8359,19 +9391,26 @@
8359	9391	tracing_off();
8360	9392
8361	9393	local_irq_save(flags);
8362		- printk_nmi_direct_enter();
8363	9394
8364	9395	/* Simulate the iterator */
8365	9396	trace_init_global_iter(&iter);
	9397	+ /* Can not use kmalloc for iter.temp */
	9398	+ iter.temp = static_temp_buf;
	9399	+ iter.temp_size = STATIC_TEMP_BUF_SIZE;
8366	9400
8367	9401	for_each_tracing_cpu(cpu) {
8368		- atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
	9402	+ atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
	9403	+ size = ring_buffer_size(iter.array_buffer->buffer, cpu);
	9404	+ trace_android_vh_ftrace_size_check(size, &ftrace_check);
8369	9405	}
8370	9406
8371	9407	old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
8372	9408
8373	9409	/* don't look at user memory in panic mode */
8374	9410	tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
	9411	+
	9412	+ if (ftrace_check)
	9413	+ goto out_enable;
8375	9414
8376	9415	switch (oops_dump_mode) {
8377	9416	case DUMP_ALL:
..	..	@@ -8396,13 +9435,14 @@
8396	9435	}
8397	9436
8398	9437	/*
8399		- * We need to stop all tracing on all CPUS to read the
	9438	+ * We need to stop all tracing on all CPUS to read
8400	9439	* the next buffer. This is a bit expensive, but is
8401	9440	* not done often. We fill all what we can read,
8402	9441	* and then release the locks again.
8403	9442	*/
8404	9443
8405	9444	while (!trace_empty(&iter)) {
	9445	+ ftrace_check = true;
8406	9446
8407	9447	if (!cnt)
8408	9448	printk(KERN_TRACE "---------------------------------\n");
..	..	@@ -8410,7 +9450,9 @@
8410	9450	cnt++;
8411	9451
8412	9452	trace_iterator_reset(&iter);
8413		- iter.iter_flags \|= TRACE_FILE_LAT_FMT;
	9453	+ trace_android_vh_ftrace_format_check(&ftrace_check);
	9454	+ if (ftrace_check)
	9455	+ iter.iter_flags \|= TRACE_FILE_LAT_FMT;
8414	9456
8415	9457	if (trace_find_next_entry_inc(&iter) != NULL) {
8416	9458	int ret;
..	..	@@ -8433,10 +9475,9 @@
8433	9475	tr->trace_flags \|= old_userobj;
8434	9476
8435	9477	for_each_tracing_cpu(cpu) {
8436		- atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
	9478	+ atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
8437	9479	}
8438	9480	atomic_dec(&dump_running);
8439		- printk_nmi_direct_exit();
8440	9481	local_irq_restore(flags);
8441	9482	}
8442	9483	EXPORT_SYMBOL_GPL(ftrace_dump);
..	..	@@ -8532,8 +9573,14 @@
8532	9573	int ring_buf_size;
8533	9574	int ret = -ENOMEM;
8534	9575
	9576	+
	9577	+ if (security_locked_down(LOCKDOWN_TRACEFS)) {
	9578	+ pr_warn("Tracing disabled due to lockdown\n");
	9579	+ return -EPERM;
	9580	+ }
	9581	+
8535	9582	/*
8536		- * Make sure we don't accidently add more trace options
	9583	+ * Make sure we don't accidentally add more trace options
8537	9584	* than we have bits for.
8538	9585	*/
8539	9586	BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
..	..	@@ -8562,7 +9609,7 @@
8562	9609
8563	9610	/*
8564	9611	* The prepare callbacks allocates some memory for the ring buffer. We
8565		- * don't free the buffer if the if the CPU goes down. If we were to free
	9612	+ * don't free the buffer if the CPU goes down. If we were to free
8566	9613	* the buffer, then the user would lose any trace that was in the
8567	9614	* buffer. The memory will be removed once the "instance" is removed.
8568	9615	*/
..	..	@@ -8582,8 +9629,7 @@
8582	9629
8583	9630	/* TODO: make the number of buffers hot pluggable with CPUS */
8584	9631	if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
8585		- printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
8586		- WARN_ON(1);
	9632	+ MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
8587	9633	goto out_free_savedcmd;
8588	9634	}
8589	9635
..	..	@@ -8628,6 +9674,7 @@
8628	9674	INIT_LIST_HEAD(&global_trace.systems);
8629	9675	INIT_LIST_HEAD(&global_trace.events);
8630	9676	INIT_LIST_HEAD(&global_trace.hist_vars);
	9677	+ INIT_LIST_HEAD(&global_trace.err_log);
8631	9678	list_add(&global_trace.list, &ftrace_trace_arrays);
8632	9679
8633	9680	apply_trace_boot_options();
..	..	@@ -8655,7 +9702,8 @@
8655	9702	if (tracepoint_printk) {
8656	9703	tracepoint_print_iter =
8657	9704	kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
8658		- if (WARN_ON(!tracepoint_print_iter))
	9705	+ if (MEM_FAIL(!tracepoint_print_iter,
	9706	+ "Failed to allocate trace iterator\n"))
8659	9707	tracepoint_printk = 0;
8660	9708	else
8661	9709	static_key_enable(&tracepoint_printk_key.key);
..	..	@@ -8695,6 +9743,11 @@
8695	9743	{
8696	9744	/* sched_clock_stable() is determined in late_initcall */
8697	9745	if (!trace_boot_clock && !sched_clock_stable()) {
	9746	+ if (security_locked_down(LOCKDOWN_TRACEFS)) {
	9747	+ pr_warn("Can not set tracing clock due to lockdown\n");
	9748	+ return -EPERM;
	9749	+ }
	9750	+
8698	9751	printk(KERN_WARNING
8699	9752	"Unstable clock detected, switching default tracing clock to \"global\"\n"
8700	9753	"If you want to keep using the local clock, then add:\n"