~hc/RK356X_SDK_RELEASE.git

..	..	@@ -11,6 +11,7 @@
11	11	#include <linux/trace_seq.h>
12	12	#include <linux/spinlock.h>
13	13	#include <linux/irq_work.h>
	14	+#include <linux/security.h>
14	15	#include <linux/uaccess.h>
15	16	#include <linux/hardirq.h>
16	17	#include <linux/kthread.h> /* for self test */
..	..	@@ -201,7 +202,7 @@
201	202	case RINGBUF_TYPE_DATA:
202	203	return rb_event_data_length(event);
203	204	default:
204		- BUG();
	205	+ WARN_ON_ONCE(1);
205	206	}
206	207	/* not hit */
207	208	return 0;
..	..	@@ -257,7 +258,7 @@
257	258	{
258	259	if (extended_time(event))
259	260	event = skip_time_extend(event);
260		- BUG_ON(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
	261	+ WARN_ON_ONCE(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
261	262	/* If length is in len field, then array[0] has the data */
262	263	if (event->type_len)
263	264	return (void *)&event->array[0];
..	..	@@ -277,6 +278,9 @@
277	278
278	279	#define for_each_buffer_cpu(buffer, cpu) \
279	280	for_each_cpu(cpu, buffer->cpumask)
	281	+
	282	+#define for_each_online_buffer_cpu(buffer, cpu) \
	283	+ for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)
280	284
281	285	#define TS_SHIFT 27
282	286	#define TS_MASK ((1ULL << TS_SHIFT) - 1)
..	..	@@ -307,8 +311,6 @@
307	311	#define RB_MISSED_EVENTS (1 << 31)
308	312	/* Missed count stored at end */
309	313	#define RB_MISSED_STORED (1 << 30)
310		-
311		-#define RB_MISSED_FLAGS (RB_MISSED_EVENTS\|RB_MISSED_STORED)
312	314
313	315	struct buffer_data_page {
314	316	u64 time_stamp; /* page time stamp */
..	..	@@ -351,20 +353,6 @@
351	353	static void rb_init_page(struct buffer_data_page *bpage)
352	354	{
353	355	local_set(&bpage->commit, 0);
354		-}
355		-
356		-/**
357		- * ring_buffer_page_len - the size of data on the page.
358		- * @page: The page to read
359		- *
360		- * Returns the amount of data on the page, including buffer page header.
361		- */
362		-size_t ring_buffer_page_len(void *page)
363		-{
364		- struct buffer_data_page *bpage = page;
365		-
366		- return (local_read(&bpage->commit) & ~RB_MISSED_FLAGS)
367		- + BUF_PAGE_HDR_SIZE;
368	356	}
369	357
370	358	/*
..	..	@@ -426,6 +414,7 @@
426	414	struct irq_work work;
427	415	wait_queue_head_t waiters;
428	416	wait_queue_head_t full_waiters;
	417	+ long wait_index;
429	418	bool waiters_pending;
430	419	bool full_waiters_pending;
431	420	bool wakeup_full;
..	..	@@ -437,11 +426,26 @@
437	426	struct rb_event_info {
438	427	u64 ts;
439	428	u64 delta;
	429	+ u64 before;
	430	+ u64 after;
440	431	unsigned long length;
441	432	struct buffer_page *tail_page;
442	433	int add_timestamp;
443	434	};
444	435
	436	+/*
	437	+ * Used for the add_timestamp
	438	+ * NONE
	439	+ * EXTEND - wants a time extend
	440	+ * ABSOLUTE - the buffer requests all events to have absolute time stamps
	441	+ * FORCE - force a full time stamp.
	442	+ */
	443	+enum {
	444	+ RB_ADD_STAMP_NONE = 0,
	445	+ RB_ADD_STAMP_EXTEND = BIT(1),
	446	+ RB_ADD_STAMP_ABSOLUTE = BIT(2),
	447	+ RB_ADD_STAMP_FORCE = BIT(3)
	448	+};
445	449	/*
446	450	* Used for which event context the event is in.
447	451	* TRANSITION = 0
..	..	@@ -461,13 +465,36 @@
461	465	RB_CTX_MAX
462	466	};
463	467
	468	+#if BITS_PER_LONG == 32
	469	+#define RB_TIME_32
	470	+#endif
	471	+
	472	+/* To test on 64 bit machines */
	473	+//#define RB_TIME_32
	474	+
	475	+#ifdef RB_TIME_32
	476	+
	477	+struct rb_time_struct {
	478	+ local_t cnt;
	479	+ local_t top;
	480	+ local_t bottom;
	481	+};
	482	+#else
	483	+#include <asm/local64.h>
	484	+struct rb_time_struct {
	485	+ local64_t time;
	486	+};
	487	+#endif
	488	+typedef struct rb_time_struct rb_time_t;
	489	+
464	490	/*
465	491	* head_page == tail_page && head == tail then buffer is empty.
466	492	*/
467	493	struct ring_buffer_per_cpu {
468	494	int cpu;
469	495	atomic_t record_disabled;
470		- struct ring_buffer *buffer;
	496	+ atomic_t resize_disabled;
	497	+ struct trace_buffer *buffer;
471	498	raw_spinlock_t reader_lock; /* serialize readers */
472	499	arch_spinlock_t lock;
473	500	struct lock_class_key lock_key;
..	..	@@ -489,9 +516,15 @@
489	516	local_t dropped_events;
490	517	local_t committing;
491	518	local_t commits;
	519	+ local_t pages_touched;
	520	+ local_t pages_lost;
	521	+ local_t pages_read;
	522	+ long last_pages_touch;
	523	+ size_t shortest_full;
492	524	unsigned long read;
493	525	unsigned long read_bytes;
494		- u64 write_stamp;
	526	+ rb_time_t write_stamp;
	527	+ rb_time_t before_stamp;
495	528	u64 read_stamp;
496	529	/* ring buffer pages to update, > 0 to add, < 0 to remove */
497	530	long nr_pages_to_update;
..	..	@@ -502,11 +535,10 @@
502	535	struct rb_irq_work irq_work;
503	536	};
504	537
505		-struct ring_buffer {
	538	+struct trace_buffer {
506	539	unsigned flags;
507	540	int cpus;
508	541	atomic_t record_disabled;
509		- atomic_t resize_disabled;
510	542	cpumask_var_t cpumask;
511	543
512	544	struct lock_class_key *reader_lock_key;
..	..	@@ -525,11 +557,256 @@
525	557	struct ring_buffer_iter {
526	558	struct ring_buffer_per_cpu *cpu_buffer;
527	559	unsigned long head;
	560	+ unsigned long next_event;
528	561	struct buffer_page *head_page;
529	562	struct buffer_page *cache_reader_page;
530	563	unsigned long cache_read;
531	564	u64 read_stamp;
	565	+ u64 page_stamp;
	566	+ struct ring_buffer_event *event;
	567	+ int missed_events;
532	568	};
	569	+
	570	+#ifdef RB_TIME_32
	571	+
	572	+/*
	573	+ * On 32 bit machines, local64_t is very expensive. As the ring
	574	+ * buffer doesn't need all the features of a true 64 bit atomic,
	575	+ * on 32 bit, it uses these functions (64 still uses local64_t).
	576	+ *
	577	+ * For the ring buffer, 64 bit required operations for the time is
	578	+ * the following:
	579	+ *
	580	+ * - Only need 59 bits (uses 60 to make it even).
	581	+ * - Reads may fail if it interrupted a modification of the time stamp.
	582	+ * It will succeed if it did not interrupt another write even if
	583	+ * the read itself is interrupted by a write.
	584	+ * It returns whether it was successful or not.
	585	+ *
	586	+ * - Writes always succeed and will overwrite other writes and writes
	587	+ * that were done by events interrupting the current write.
	588	+ *
	589	+ * - A write followed by a read of the same time stamp will always succeed,
	590	+ * but may not contain the same value.
	591	+ *
	592	+ * - A cmpxchg will fail if it interrupted another write or cmpxchg.
	593	+ * Other than that, it acts like a normal cmpxchg.
	594	+ *
	595	+ * The 60 bit time stamp is broken up by 30 bits in a top and bottom half
	596	+ * (bottom being the least significant 30 bits of the 60 bit time stamp).
	597	+ *
	598	+ * The two most significant bits of each half holds a 2 bit counter (0-3).
	599	+ * Each update will increment this counter by one.
	600	+ * When reading the top and bottom, if the two counter bits match then the
	601	+ * top and bottom together make a valid 60 bit number.
	602	+ */
	603	+#define RB_TIME_SHIFT 30
	604	+#define RB_TIME_VAL_MASK ((1 << RB_TIME_SHIFT) - 1)
	605	+
	606	+static inline int rb_time_cnt(unsigned long val)
	607	+{
	608	+ return (val >> RB_TIME_SHIFT) & 3;
	609	+}
	610	+
	611	+static inline u64 rb_time_val(unsigned long top, unsigned long bottom)
	612	+{
	613	+ u64 val;
	614	+
	615	+ val = top & RB_TIME_VAL_MASK;
	616	+ val <<= RB_TIME_SHIFT;
	617	+ val \|= bottom & RB_TIME_VAL_MASK;
	618	+
	619	+ return val;
	620	+}
	621	+
	622	+static inline bool __rb_time_read(rb_time_t t, u64 ret, unsigned long *cnt)
	623	+{
	624	+ unsigned long top, bottom;
	625	+ unsigned long c;
	626	+
	627	+ /*
	628	+ * If the read is interrupted by a write, then the cnt will
	629	+ * be different. Loop until both top and bottom have been read
	630	+ * without interruption.
	631	+ */
	632	+ do {
	633	+ c = local_read(&t->cnt);
	634	+ top = local_read(&t->top);
	635	+ bottom = local_read(&t->bottom);
	636	+ } while (c != local_read(&t->cnt));
	637	+
	638	+ *cnt = rb_time_cnt(top);
	639	+
	640	+ /* If top and bottom counts don't match, this interrupted a write */
	641	+ if (*cnt != rb_time_cnt(bottom))
	642	+ return false;
	643	+
	644	+ *ret = rb_time_val(top, bottom);
	645	+ return true;
	646	+}
	647	+
	648	+static bool rb_time_read(rb_time_t t, u64 ret)
	649	+{
	650	+ unsigned long cnt;
	651	+
	652	+ return __rb_time_read(t, ret, &cnt);
	653	+}
	654	+
	655	+static inline unsigned long rb_time_val_cnt(unsigned long val, unsigned long cnt)
	656	+{
	657	+ return (val & RB_TIME_VAL_MASK) \| ((cnt & 3) << RB_TIME_SHIFT);
	658	+}
	659	+
	660	+static inline void rb_time_split(u64 val, unsigned long top, unsigned long bottom)
	661	+{
	662	+ *top = (unsigned long)((val >> RB_TIME_SHIFT) & RB_TIME_VAL_MASK);
	663	+ *bottom = (unsigned long)(val & RB_TIME_VAL_MASK);
	664	+}
	665	+
	666	+static inline void rb_time_val_set(local_t *t, unsigned long val, unsigned long cnt)
	667	+{
	668	+ val = rb_time_val_cnt(val, cnt);
	669	+ local_set(t, val);
	670	+}
	671	+
	672	+static void rb_time_set(rb_time_t *t, u64 val)
	673	+{
	674	+ unsigned long cnt, top, bottom;
	675	+
	676	+ rb_time_split(val, &top, &bottom);
	677	+
	678	+ /* Writes always succeed with a valid number even if it gets interrupted. */
	679	+ do {
	680	+ cnt = local_inc_return(&t->cnt);
	681	+ rb_time_val_set(&t->top, top, cnt);
	682	+ rb_time_val_set(&t->bottom, bottom, cnt);
	683	+ } while (cnt != local_read(&t->cnt));
	684	+}
	685	+
	686	+static inline bool
	687	+rb_time_read_cmpxchg(local_t *l, unsigned long expect, unsigned long set)
	688	+{
	689	+ unsigned long ret;
	690	+
	691	+ ret = local_cmpxchg(l, expect, set);
	692	+ return ret == expect;
	693	+}
	694	+
	695	+static int rb_time_cmpxchg(rb_time_t *t, u64 expect, u64 set)
	696	+{
	697	+ unsigned long cnt, top, bottom;
	698	+ unsigned long cnt2, top2, bottom2;
	699	+ u64 val;
	700	+
	701	+ /* The cmpxchg always fails if it interrupted an update */
	702	+ if (!__rb_time_read(t, &val, &cnt2))
	703	+ return false;
	704	+
	705	+ if (val != expect)
	706	+ return false;
	707	+
	708	+ cnt = local_read(&t->cnt);
	709	+ if ((cnt & 3) != cnt2)
	710	+ return false;
	711	+
	712	+ cnt2 = cnt + 1;
	713	+
	714	+ rb_time_split(val, &top, &bottom);
	715	+ top = rb_time_val_cnt(top, cnt);
	716	+ bottom = rb_time_val_cnt(bottom, cnt);
	717	+
	718	+ rb_time_split(set, &top2, &bottom2);
	719	+ top2 = rb_time_val_cnt(top2, cnt2);
	720	+ bottom2 = rb_time_val_cnt(bottom2, cnt2);
	721	+
	722	+ if (!rb_time_read_cmpxchg(&t->cnt, cnt, cnt2))
	723	+ return false;
	724	+ if (!rb_time_read_cmpxchg(&t->top, top, top2))
	725	+ return false;
	726	+ if (!rb_time_read_cmpxchg(&t->bottom, bottom, bottom2))
	727	+ return false;
	728	+ return true;
	729	+}
	730	+
	731	+#else /* 64 bits */
	732	+
	733	+/* local64_t always succeeds */
	734	+
	735	+static inline bool rb_time_read(rb_time_t t, u64 ret)
	736	+{
	737	+ *ret = local64_read(&t->time);
	738	+ return true;
	739	+}
	740	+static void rb_time_set(rb_time_t *t, u64 val)
	741	+{
	742	+ local64_set(&t->time, val);
	743	+}
	744	+
	745	+static bool rb_time_cmpxchg(rb_time_t *t, u64 expect, u64 set)
	746	+{
	747	+ u64 val;
	748	+ val = local64_cmpxchg(&t->time, expect, set);
	749	+ return val == expect;
	750	+}
	751	+#endif
	752	+
	753	+/**
	754	+ * ring_buffer_nr_pages - get the number of buffer pages in the ring buffer
	755	+ * @buffer: The ring_buffer to get the number of pages from
	756	+ * @cpu: The cpu of the ring_buffer to get the number of pages from
	757	+ *
	758	+ * Returns the number of pages used by a per_cpu buffer of the ring buffer.
	759	+ */
	760	+size_t ring_buffer_nr_pages(struct trace_buffer *buffer, int cpu)
	761	+{
	762	+ return buffer->buffers[cpu]->nr_pages;
	763	+}
	764	+
	765	+/**
	766	+ * ring_buffer_nr_pages_dirty - get the number of used pages in the ring buffer
	767	+ * @buffer: The ring_buffer to get the number of pages from
	768	+ * @cpu: The cpu of the ring_buffer to get the number of pages from
	769	+ *
	770	+ * Returns the number of pages that have content in the ring buffer.
	771	+ */
	772	+size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu)
	773	+{
	774	+ size_t read;
	775	+ size_t lost;
	776	+ size_t cnt;
	777	+
	778	+ read = local_read(&buffer->buffers[cpu]->pages_read);
	779	+ lost = local_read(&buffer->buffers[cpu]->pages_lost);
	780	+ cnt = local_read(&buffer->buffers[cpu]->pages_touched);
	781	+
	782	+ if (WARN_ON_ONCE(cnt < lost))
	783	+ return 0;
	784	+
	785	+ cnt -= lost;
	786	+
	787	+ /* The reader can read an empty page, but not more than that */
	788	+ if (cnt < read) {
	789	+ WARN_ON_ONCE(read > cnt + 1);
	790	+ return 0;
	791	+ }
	792	+
	793	+ return cnt - read;
	794	+}
	795	+
	796	+static __always_inline bool full_hit(struct trace_buffer *buffer, int cpu, int full)
	797	+{
	798	+ struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	799	+ size_t nr_pages;
	800	+ size_t dirty;
	801	+
	802	+ nr_pages = cpu_buffer->nr_pages;
	803	+ if (!nr_pages \|\| !full)
	804	+ return true;
	805	+
	806	+ dirty = ring_buffer_nr_dirty_pages(buffer, cpu);
	807	+
	808	+ return (dirty * 100) > (full * nr_pages);
	809	+}
533	810
534	811	/*
535	812	* rb_wake_up_waiters - wake up tasks waiting for ring buffer input
..	..	@@ -542,27 +819,60 @@
542	819	struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);
543	820
544	821	wake_up_all(&rbwork->waiters);
545		- if (rbwork->wakeup_full) {
	822	+ if (rbwork->full_waiters_pending \|\| rbwork->wakeup_full) {
546	823	rbwork->wakeup_full = false;
	824	+ rbwork->full_waiters_pending = false;
547	825	wake_up_all(&rbwork->full_waiters);
548	826	}
	827	+}
	828	+
	829	+/**
	830	+ * ring_buffer_wake_waiters - wake up any waiters on this ring buffer
	831	+ * @buffer: The ring buffer to wake waiters on
	832	+ *
	833	+ * In the case of a file that represents a ring buffer is closing,
	834	+ * it is prudent to wake up any waiters that are on this.
	835	+ */
	836	+void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu)
	837	+{
	838	+ struct ring_buffer_per_cpu *cpu_buffer;
	839	+ struct rb_irq_work *rbwork;
	840	+
	841	+ if (cpu == RING_BUFFER_ALL_CPUS) {
	842	+
	843	+ /* Wake up individual ones too. One level recursion */
	844	+ for_each_buffer_cpu(buffer, cpu)
	845	+ ring_buffer_wake_waiters(buffer, cpu);
	846	+
	847	+ rbwork = &buffer->irq_work;
	848	+ } else {
	849	+ cpu_buffer = buffer->buffers[cpu];
	850	+ rbwork = &cpu_buffer->irq_work;
	851	+ }
	852	+
	853	+ rbwork->wait_index++;
	854	+ /* make sure the waiters see the new index */
	855	+ smp_wmb();
	856	+
	857	+ rb_wake_up_waiters(&rbwork->work);
549	858	}
550	859
551	860	/**
552	861	* ring_buffer_wait - wait for input to the ring buffer
553	862	* @buffer: buffer to wait on
554	863	* @cpu: the cpu buffer to wait on
555		- * @full: wait until a full page is available, if @cpu != RING_BUFFER_ALL_CPUS
	864	+ * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS
556	865	*
557	866	* If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
558	867	* as data is added to any of the @buffer's cpu buffers. Otherwise
559	868	* it will wait for data to be added to a specific cpu buffer.
560	869	*/
561		-int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
	870	+int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full)
562	871	{
563		- struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer);
	872	+ struct ring_buffer_per_cpu *cpu_buffer;
564	873	DEFINE_WAIT(wait);
565	874	struct rb_irq_work *work;
	875	+ long wait_index;
566	876	int ret = 0;
567	877
568	878	/*
..	..	@@ -573,7 +883,7 @@
573	883	if (cpu == RING_BUFFER_ALL_CPUS) {
574	884	work = &buffer->irq_work;
575	885	/* Full only makes sense on per cpu reads */
576		- full = false;
	886	+ full = 0;
577	887	} else {
578	888	if (!cpumask_test_cpu(cpu, buffer->cpumask))
579	889	return -ENODEV;
..	..	@@ -581,6 +891,7 @@
581	891	work = &cpu_buffer->irq_work;
582	892	}
583	893
	894	+ wait_index = READ_ONCE(work->wait_index);
584	895
585	896	while (true) {
586	897	if (full)
..	..	@@ -625,19 +936,29 @@
625	936	!ring_buffer_empty_cpu(buffer, cpu)) {
626	937	unsigned long flags;
627	938	bool pagebusy;
	939	+ bool done;
628	940
629	941	if (!full)
630	942	break;
631	943
632	944	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
633	945	pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
634		- raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
	946	+ done = !pagebusy && full_hit(buffer, cpu, full);
635	947
636		- if (!pagebusy)
	948	+ if (!cpu_buffer->shortest_full \|\|
	949	+ cpu_buffer->shortest_full > full)
	950	+ cpu_buffer->shortest_full = full;
	951	+ raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
	952	+ if (done)
637	953	break;
638	954	}
639	955
640	956	schedule();
	957	+
	958	+ /* Make sure to see the new wait index */
	959	+ smp_rmb();
	960	+ if (wait_index != work->wait_index)
	961	+ break;
641	962	}
642	963
643	964	if (full)
..	..	@@ -654,6 +975,7 @@
654	975	* @cpu: the cpu buffer to wait on
655	976	* @filp: the file descriptor
656	977	* @poll_table: The poll descriptor
	978	+ * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS
657	979	*
658	980	* If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
659	981	* as data is added to any of the @buffer's cpu buffers. Otherwise
..	..	@@ -662,15 +984,16 @@
662	984	* Returns EPOLLIN \| EPOLLRDNORM if data exists in the buffers,
663	985	* zero otherwise.
664	986	*/
665		-__poll_t ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu,
666		- struct file filp, poll_table poll_table)
	987	+__poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu,
	988	+ struct file filp, poll_table poll_table, int full)
667	989	{
668	990	struct ring_buffer_per_cpu *cpu_buffer;
669	991	struct rb_irq_work *work;
670	992
671		- if (cpu == RING_BUFFER_ALL_CPUS)
	993	+ if (cpu == RING_BUFFER_ALL_CPUS) {
672	994	work = &buffer->irq_work;
673		- else {
	995	+ full = 0;
	996	+ } else {
674	997	if (!cpumask_test_cpu(cpu, buffer->cpumask))
675	998	return -EINVAL;
676	999
..	..	@@ -678,8 +1001,14 @@
678	1001	work = &cpu_buffer->irq_work;
679	1002	}
680	1003
681		- poll_wait(filp, &work->waiters, poll_table);
682		- work->waiters_pending = true;
	1004	+ if (full) {
	1005	+ poll_wait(filp, &work->full_waiters, poll_table);
	1006	+ work->full_waiters_pending = true;
	1007	+ } else {
	1008	+ poll_wait(filp, &work->waiters, poll_table);
	1009	+ work->waiters_pending = true;
	1010	+ }
	1011	+
683	1012	/*
684	1013	* There's a tight race between setting the waiters_pending and
685	1014	* checking if the ring buffer is empty. Once the waiters_pending bit
..	..	@@ -694,6 +1023,9 @@
694	1023	* will fix it later.
695	1024	*/
696	1025	smp_mb();
	1026	+
	1027	+ if (full)
	1028	+ return full_hit(buffer, cpu, full) ? EPOLLIN \| EPOLLRDNORM : 0;
697	1029
698	1030	if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) \|\|
699	1031	(cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))
..	..	@@ -720,13 +1052,21 @@
720	1052	/* Up this if you want to test the TIME_EXTENTS and normalization */
721	1053	#define DEBUG_SHIFT 0
722	1054
723		-static inline u64 rb_time_stamp(struct ring_buffer *buffer)
	1055	+static inline u64 rb_time_stamp(struct trace_buffer *buffer)
724	1056	{
	1057	+ u64 ts;
	1058	+
	1059	+ /* Skip retpolines :-( */
	1060	+ if (IS_ENABLED(CONFIG_RETPOLINE) && likely(buffer->clock == trace_clock_local))
	1061	+ ts = trace_clock_local();
	1062	+ else
	1063	+ ts = buffer->clock();
	1064	+
725	1065	/* shift to debug/test normalization and TIME_EXTENTS */
726		- return buffer->clock() << DEBUG_SHIFT;
	1066	+ return ts << DEBUG_SHIFT;
727	1067	}
728	1068
729		-u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu)
	1069	+u64 ring_buffer_time_stamp(struct trace_buffer *buffer, int cpu)
730	1070	{
731	1071	u64 time;
732	1072
..	..	@@ -738,7 +1078,7 @@
738	1078	}
739	1079	EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);
740	1080
741		-void ring_buffer_normalize_time_stamp(struct ring_buffer *buffer,
	1081	+void ring_buffer_normalize_time_stamp(struct trace_buffer *buffer,
742	1082	int cpu, u64 *ts)
743	1083	{
744	1084	/* Just stupid testing the normalize function and deltas */
..	..	@@ -1056,6 +1396,7 @@
1056	1396	old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
1057	1397	old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);
1058	1398
	1399	+ local_inc(&cpu_buffer->pages_touched);
1059	1400	/*
1060	1401	* Just make sure we have seen our old_write and synchronize
1061	1402	* with any interrupts that come in.
..	..	@@ -1260,7 +1601,7 @@
1260	1601	}
1261	1602
1262	1603	static struct ring_buffer_per_cpu *
1263		-rb_allocate_cpu_buffer(struct ring_buffer *buffer, long nr_pages, int cpu)
	1604	+rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu)
1264	1605	{
1265	1606	struct ring_buffer_per_cpu *cpu_buffer;
1266	1607	struct buffer_page *bpage;
..	..	@@ -1327,9 +1668,9 @@
1327	1668
1328	1669	free_buffer_page(cpu_buffer->reader_page);
1329	1670
1330		- rb_head_page_deactivate(cpu_buffer);
1331		-
1332	1671	if (head) {
	1672	+ rb_head_page_deactivate(cpu_buffer);
	1673	+
1333	1674	list_for_each_entry_safe(bpage, tmp, head, list) {
1334	1675	list_del_init(&bpage->list);
1335	1676	free_buffer_page(bpage);
..	..	@@ -1345,16 +1686,17 @@
1345	1686	* __ring_buffer_alloc - allocate a new ring_buffer
1346	1687	* @size: the size in bytes per cpu that is needed.
1347	1688	* @flags: attributes to set for the ring buffer.
	1689	+ * @key: ring buffer reader_lock_key.
1348	1690	*
1349	1691	* Currently the only flag that is available is the RB_FL_OVERWRITE
1350	1692	* flag. This flag means that the buffer will overwrite old data
1351	1693	* when the buffer wraps. If this flag is not set, the buffer will
1352	1694	* drop data when the tail hits the head.
1353	1695	*/
1354		-struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
	1696	+struct trace_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
1355	1697	struct lock_class_key *key)
1356	1698	{
1357		- struct ring_buffer *buffer;
	1699	+ struct trace_buffer *buffer;
1358	1700	long nr_pages;
1359	1701	int bsize;
1360	1702	int cpu;
..	..	@@ -1424,7 +1766,7 @@
1424	1766	* @buffer: the buffer to free.
1425	1767	*/
1426	1768	void
1427		-ring_buffer_free(struct ring_buffer *buffer)
	1769	+ring_buffer_free(struct trace_buffer *buffer)
1428	1770	{
1429	1771	int cpu;
1430	1772
..	..	@@ -1440,18 +1782,18 @@
1440	1782	}
1441	1783	EXPORT_SYMBOL_GPL(ring_buffer_free);
1442	1784
1443		-void ring_buffer_set_clock(struct ring_buffer *buffer,
	1785	+void ring_buffer_set_clock(struct trace_buffer *buffer,
1444	1786	u64 (*clock)(void))
1445	1787	{
1446	1788	buffer->clock = clock;
1447	1789	}
1448	1790
1449		-void ring_buffer_set_time_stamp_abs(struct ring_buffer *buffer, bool abs)
	1791	+void ring_buffer_set_time_stamp_abs(struct trace_buffer *buffer, bool abs)
1450	1792	{
1451	1793	buffer->time_stamp_abs = abs;
1452	1794	}
1453	1795
1454		-bool ring_buffer_time_stamp_abs(struct ring_buffer *buffer)
	1796	+bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer)
1455	1797	{
1456	1798	return buffer->time_stamp_abs;
1457	1799	}
..	..	@@ -1564,6 +1906,7 @@
1564	1906	*/
1565	1907	local_add(page_entries, &cpu_buffer->overrun);
1566	1908	local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
	1909	+ local_inc(&cpu_buffer->pages_lost);
1567	1910	}
1568	1911
1569	1912	/*
..	..	@@ -1689,7 +2032,7 @@
1689	2032	*
1690	2033	* Returns 0 on success and < 0 on failure.
1691	2034	*/
1692		-int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size,
	2035	+int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size,
1693	2036	int cpu_id)
1694	2037	{
1695	2038	struct ring_buffer_per_cpu *cpu_buffer;
..	..	@@ -1715,18 +2058,24 @@
1715	2058
1716	2059	size = nr_pages * BUF_PAGE_SIZE;
1717	2060
1718		- /*
1719		- * Don't succeed if resizing is disabled, as a reader might be
1720		- * manipulating the ring buffer and is expecting a sane state while
1721		- * this is true.
1722		- */
1723		- if (atomic_read(&buffer->resize_disabled))
1724		- return -EBUSY;
1725		-
1726	2061	/* prevent another thread from changing buffer sizes */
1727	2062	mutex_lock(&buffer->mutex);
1728	2063
	2064	+
1729	2065	if (cpu_id == RING_BUFFER_ALL_CPUS) {
	2066	+ /*
	2067	+ * Don't succeed if resizing is disabled, as a reader might be
	2068	+ * manipulating the ring buffer and is expecting a sane state while
	2069	+ * this is true.
	2070	+ */
	2071	+ for_each_buffer_cpu(buffer, cpu) {
	2072	+ cpu_buffer = buffer->buffers[cpu];
	2073	+ if (atomic_read(&cpu_buffer->resize_disabled)) {
	2074	+ err = -EBUSY;
	2075	+ goto out_err_unlock;
	2076	+ }
	2077	+ }
	2078	+
1730	2079	/* calculate the pages to update */
1731	2080	for_each_buffer_cpu(buffer, cpu) {
1732	2081	cpu_buffer = buffer->buffers[cpu];
..	..	@@ -1794,6 +2143,16 @@
1794	2143	if (nr_pages == cpu_buffer->nr_pages)
1795	2144	goto out;
1796	2145
	2146	+ /*
	2147	+ * Don't succeed if resizing is disabled, as a reader might be
	2148	+ * manipulating the ring buffer and is expecting a sane state while
	2149	+ * this is true.
	2150	+ */
	2151	+ if (atomic_read(&cpu_buffer->resize_disabled)) {
	2152	+ err = -EBUSY;
	2153	+ goto out_err_unlock;
	2154	+ }
	2155	+
1797	2156	cpu_buffer->nr_pages_to_update = nr_pages -
1798	2157	cpu_buffer->nr_pages;
1799	2158
..	..	@@ -1836,7 +2195,7 @@
1836	2195	* There could have been a race between checking
1837	2196	* record_disable and incrementing it.
1838	2197	*/
1839		- synchronize_sched();
	2198	+ synchronize_rcu();
1840	2199	for_each_buffer_cpu(buffer, cpu) {
1841	2200	cpu_buffer = buffer->buffers[cpu];
1842	2201	rb_check_pages(cpu_buffer);
..	..	@@ -1863,12 +2222,13 @@
1863	2222	free_buffer_page(bpage);
1864	2223	}
1865	2224	}
	2225	+ out_err_unlock:
1866	2226	mutex_unlock(&buffer->mutex);
1867	2227	return err;
1868	2228	}
1869	2229	EXPORT_SYMBOL_GPL(ring_buffer_resize);
1870	2230
1871		-void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val)
	2231	+void ring_buffer_change_overwrite(struct trace_buffer *buffer, int val)
1872	2232	{
1873	2233	mutex_lock(&buffer->mutex);
1874	2234	if (val)
..	..	@@ -1891,15 +2251,63 @@
1891	2251	cpu_buffer->reader_page->read);
1892	2252	}
1893	2253
1894		-static __always_inline struct ring_buffer_event *
1895		-rb_iter_head_event(struct ring_buffer_iter *iter)
1896		-{
1897		- return __rb_page_index(iter->head_page, iter->head);
1898		-}
1899		-
1900	2254	static __always_inline unsigned rb_page_commit(struct buffer_page *bpage)
1901	2255	{
1902	2256	return local_read(&bpage->page->commit);
	2257	+}
	2258	+
	2259	+static struct ring_buffer_event *
	2260	+rb_iter_head_event(struct ring_buffer_iter *iter)
	2261	+{
	2262	+ struct ring_buffer_event *event;
	2263	+ struct buffer_page *iter_head_page = iter->head_page;
	2264	+ unsigned long commit;
	2265	+ unsigned length;
	2266	+
	2267	+ if (iter->head != iter->next_event)
	2268	+ return iter->event;
	2269	+
	2270	+ /*
	2271	+ * When the writer goes across pages, it issues a cmpxchg which
	2272	+ * is a mb(), which will synchronize with the rmb here.
	2273	+ * (see rb_tail_page_update() and __rb_reserve_next())
	2274	+ */
	2275	+ commit = rb_page_commit(iter_head_page);
	2276	+ smp_rmb();
	2277	+ event = __rb_page_index(iter_head_page, iter->head);
	2278	+ length = rb_event_length(event);
	2279	+
	2280	+ /*
	2281	+ * READ_ONCE() doesn't work on functions and we don't want the
	2282	+ * compiler doing any crazy optimizations with length.
	2283	+ */
	2284	+ barrier();
	2285	+
	2286	+ if ((iter->head + length) > commit \|\| length > BUF_MAX_DATA_SIZE)
	2287	+ /* Writer corrupted the read? */
	2288	+ goto reset;
	2289	+
	2290	+ memcpy(iter->event, event, length);
	2291	+ /*
	2292	+ * If the page stamp is still the same after this rmb() then the
	2293	+ * event was safely copied without the writer entering the page.
	2294	+ */
	2295	+ smp_rmb();
	2296	+
	2297	+ /* Make sure the page didn't change since we read this */
	2298	+ if (iter->page_stamp != iter_head_page->page->time_stamp \|\|
	2299	+ commit > rb_page_commit(iter_head_page))
	2300	+ goto reset;
	2301	+
	2302	+ iter->next_event = iter->head + length;
	2303	+ return iter->event;
	2304	+ reset:
	2305	+ /* Reset to the beginning */
	2306	+ iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp;
	2307	+ iter->head = 0;
	2308	+ iter->next_event = 0;
	2309	+ iter->missed_events = 1;
	2310	+ return NULL;
1903	2311	}
1904	2312
1905	2313	/* Size is determined by what has been committed */
..	..	@@ -1937,8 +2345,9 @@
1937	2345	else
1938	2346	rb_inc_page(cpu_buffer, &iter->head_page);
1939	2347
1940		- iter->read_stamp = iter->head_page->page->time_stamp;
	2348	+ iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp;
1941	2349	iter->head = 0;
	2350	+ iter->next_event = 0;
1942	2351	}
1943	2352
1944	2353	/*
..	..	@@ -1988,6 +2397,7 @@
1988	2397	*/
1989	2398	local_add(entries, &cpu_buffer->overrun);
1990	2399	local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
	2400	+ local_inc(&cpu_buffer->pages_lost);
1991	2401
1992	2402	/*
1993	2403	* The entries will be zeroed out when we move the
..	..	@@ -2156,6 +2566,9 @@
2156	2566	/* Mark the rest of the page with padding */
2157	2567	rb_event_set_padding(event);
2158	2568
	2569	+ /* Make sure the padding is visible before the write update */
	2570	+ smp_wmb();
	2571	+
2159	2572	/* Set the write back to the previous setting */
2160	2573	local_sub(length, &tail_page->write);
2161	2574	return;
..	..	@@ -2166,6 +2579,9 @@
2166	2579	event->type_len = RINGBUF_TYPE_PADDING;
2167	2580	/* time delta must be non zero */
2168	2581	event->time_delta = 1;
	2582	+
	2583	+ /* Make sure the padding is visible before the tail_page->write update */
	2584	+ smp_wmb();
2169	2585
2170	2586	/* Set write to end of buffer */
2171	2587	length = (tail + length) - BUF_PAGE_SIZE;
..	..	@@ -2183,7 +2599,7 @@
2183	2599	{
2184	2600	struct buffer_page *tail_page = info->tail_page;
2185	2601	struct buffer_page *commit_page = cpu_buffer->commit_page;
2186		- struct ring_buffer *buffer = cpu_buffer->buffer;
	2602	+ struct trace_buffer *buffer = cpu_buffer->buffer;
2187	2603	struct buffer_page *next_page;
2188	2604	int ret;
2189	2605
..	..	@@ -2280,8 +2696,8 @@
2280	2696	return NULL;
2281	2697	}
2282	2698
2283		-/* Slow path, do not inline */
2284		-static noinline struct ring_buffer_event *
	2699	+/* Slow path */
	2700	+static struct ring_buffer_event *
2285	2701	rb_add_time_stamp(struct ring_buffer_event *event, u64 delta, bool abs)
2286	2702	{
2287	2703	if (abs)
..	..	@@ -2305,13 +2721,73 @@
2305	2721	static inline bool rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
2306	2722	struct ring_buffer_event *event);
2307	2723
	2724	+#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
	2725	+static inline bool sched_clock_stable(void)
	2726	+{
	2727	+ return true;
	2728	+}
	2729	+#endif
	2730	+
	2731	+static void
	2732	+rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
	2733	+ struct rb_event_info *info)
	2734	+{
	2735	+ u64 write_stamp;
	2736	+
	2737	+ WARN_ONCE(1, "Delta way too big! %llu ts=%llu before=%llu after=%llu write stamp=%llu\n%s",
	2738	+ (unsigned long long)info->delta,
	2739	+ (unsigned long long)info->ts,
	2740	+ (unsigned long long)info->before,
	2741	+ (unsigned long long)info->after,
	2742	+ (unsigned long long)(rb_time_read(&cpu_buffer->write_stamp, &write_stamp) ? write_stamp : 0),
	2743	+ sched_clock_stable() ? "" :
	2744	+ "If you just came from a suspend/resume,\n"
	2745	+ "please switch to the trace global clock:\n"
	2746	+ " echo global > /sys/kernel/debug/tracing/trace_clock\n"
	2747	+ "or add trace_clock=global to the kernel command line\n");
	2748	+}
	2749	+
	2750	+static void rb_add_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
	2751	+ struct ring_buffer_event **event,
	2752	+ struct rb_event_info *info,
	2753	+ u64 *delta,
	2754	+ unsigned int *length)
	2755	+{
	2756	+ bool abs = info->add_timestamp &
	2757	+ (RB_ADD_STAMP_FORCE \| RB_ADD_STAMP_ABSOLUTE);
	2758	+
	2759	+ if (unlikely(info->delta > (1ULL << 59))) {
	2760	+ /* did the clock go backwards */
	2761	+ if (info->before == info->after && info->before > info->ts) {
	2762	+ /* not interrupted */
	2763	+ static int once;
	2764	+
	2765	+ /*
	2766	+ * This is possible with a recalibrating of the TSC.
	2767	+ * Do not produce a call stack, but just report it.
	2768	+ */
	2769	+ if (!once) {
	2770	+ once++;
	2771	+ pr_warn("Ring buffer clock went backwards: %llu -> %llu\n",
	2772	+ info->before, info->ts);
	2773	+ }
	2774	+ } else
	2775	+ rb_check_timestamp(cpu_buffer, info);
	2776	+ if (!abs)
	2777	+ info->delta = 0;
	2778	+ }
	2779	+ event = rb_add_time_stamp(event, info->delta, abs);
	2780	+ *length -= RB_LEN_TIME_EXTEND;
	2781	+ *delta = 0;
	2782	+}
	2783	+
2308	2784	/**
2309	2785	* rb_update_event - update event type and data
	2786	+ * @cpu_buffer: The per cpu buffer of the @event
2310	2787	* @event: the event to update
2311		- * @type: the type of event
2312		- * @length: the size of the event field in the ring buffer
	2788	+ * @info: The info to update the @event with (contains length and delta)
2313	2789	*
2314		- * Update the type and data fields of the event. The length
	2790	+ * Update the type and data fields of the @event. The length
2315	2791	* is the actual size that is written to the ring buffer,
2316	2792	* and with this, we can determine what to place into the
2317	2793	* data field.
..	..	@@ -2324,21 +2800,12 @@
2324	2800	unsigned length = info->length;
2325	2801	u64 delta = info->delta;
2326	2802
2327		- /* Only a commit updates the timestamp */
2328		- if (unlikely(!rb_event_is_commit(cpu_buffer, event)))
2329		- delta = 0;
2330		-
2331	2803	/*
2332	2804	* If we need to add a timestamp, then we
2333	2805	* add it to the start of the reserved space.
2334	2806	*/
2335		- if (unlikely(info->add_timestamp)) {
2336		- bool abs = ring_buffer_time_stamp_abs(cpu_buffer->buffer);
2337		-
2338		- event = rb_add_time_stamp(event, abs ? info->delta : delta, abs);
2339		- length -= RB_LEN_TIME_EXTEND;
2340		- delta = 0;
2341		- }
	2807	+ if (unlikely(info->add_timestamp))
	2808	+ rb_add_timestamp(cpu_buffer, &event, info, &delta, &length);
2342	2809
2343	2810	event->time_delta = delta;
2344	2811	length -= RB_EVNT_HDR_SIZE;
..	..	@@ -2381,12 +2848,38 @@
2381	2848	return length;
2382	2849	}
2383	2850
2384		-#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
2385		-static inline bool sched_clock_stable(void)
	2851	+static __always_inline bool
	2852	+rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
	2853	+ struct ring_buffer_event *event)
2386	2854	{
2387		- return true;
	2855	+ unsigned long addr = (unsigned long)event;
	2856	+ unsigned long index;
	2857	+
	2858	+ index = rb_event_index(event);
	2859	+ addr &= PAGE_MASK;
	2860	+
	2861	+ return cpu_buffer->commit_page->page == (void *)addr &&
	2862	+ rb_commit_index(cpu_buffer) == index;
2388	2863	}
2389		-#endif
	2864	+
	2865	+static u64 rb_time_delta(struct ring_buffer_event *event)
	2866	+{
	2867	+ switch (event->type_len) {
	2868	+ case RINGBUF_TYPE_PADDING:
	2869	+ return 0;
	2870	+
	2871	+ case RINGBUF_TYPE_TIME_EXTEND:
	2872	+ return ring_buffer_event_time_stamp(event);
	2873	+
	2874	+ case RINGBUF_TYPE_TIME_STAMP:
	2875	+ return 0;
	2876	+
	2877	+ case RINGBUF_TYPE_DATA:
	2878	+ return event->time_delta;
	2879	+ default:
	2880	+ return 0;
	2881	+ }
	2882	+}
2390	2883
2391	2884	static inline int
2392	2885	rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
..	..	@@ -2396,6 +2889,8 @@
2396	2889	struct buffer_page *bpage;
2397	2890	unsigned long index;
2398	2891	unsigned long addr;
	2892	+ u64 write_stamp;
	2893	+ u64 delta;
2399	2894
2400	2895	new_index = rb_event_index(event);
2401	2896	old_index = new_index + rb_event_ts_length(event);
..	..	@@ -2404,10 +2899,43 @@
2404	2899
2405	2900	bpage = READ_ONCE(cpu_buffer->tail_page);
2406	2901
	2902	+ delta = rb_time_delta(event);
	2903	+
	2904	+ if (!rb_time_read(&cpu_buffer->write_stamp, &write_stamp))
	2905	+ return 0;
	2906	+
	2907	+ /* Make sure the write stamp is read before testing the location */
	2908	+ barrier();
	2909	+
2407	2910	if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
2408	2911	unsigned long write_mask =
2409	2912	local_read(&bpage->write) & ~RB_WRITE_MASK;
2410	2913	unsigned long event_length = rb_event_length(event);
	2914	+
	2915	+ /* Something came in, can't discard */
	2916	+ if (!rb_time_cmpxchg(&cpu_buffer->write_stamp,
	2917	+ write_stamp, write_stamp - delta))
	2918	+ return 0;
	2919	+
	2920	+ /*
	2921	+ * It's possible that the event time delta is zero
	2922	+ * (has the same time stamp as the previous event)
	2923	+ * in which case write_stamp and before_stamp could
	2924	+ * be the same. In such a case, force before_stamp
	2925	+ * to be different than write_stamp. It doesn't
	2926	+ * matter what it is, as long as its different.
	2927	+ */
	2928	+ if (!delta)
	2929	+ rb_time_set(&cpu_buffer->before_stamp, 0);
	2930	+
	2931	+ /*
	2932	+ * If an event were to come in now, it would see that the
	2933	+ * write_stamp and the before_stamp are different, and assume
	2934	+ * that this event just added itself before updating
	2935	+ * the write stamp. The interrupting event will fix the
	2936	+ * write stamp for us, and use the before stamp as its delta.
	2937	+ */
	2938	+
2411	2939	/*
2412	2940	* This is on the tail page. It is possible that
2413	2941	* a write could come in and move the tail page
..	..	@@ -2459,10 +2987,6 @@
2459	2987	local_set(&cpu_buffer->commit_page->page->commit,
2460	2988	rb_page_write(cpu_buffer->commit_page));
2461	2989	rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
2462		- /* Only update the write stamp if the page has an event */
2463		- if (rb_page_write(cpu_buffer->commit_page))
2464		- cpu_buffer->write_stamp =
2465		- cpu_buffer->commit_page->page->time_stamp;
2466	2990	/* add barrier to keep gcc from optimizing too much */
2467	2991	barrier();
2468	2992	}
..	..	@@ -2534,62 +3058,16 @@
2534	3058	event->time_delta = 1;
2535	3059	}
2536	3060
2537		-static __always_inline bool
2538		-rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
2539		- struct ring_buffer_event *event)
2540		-{
2541		- unsigned long addr = (unsigned long)event;
2542		- unsigned long index;
2543		-
2544		- index = rb_event_index(event);
2545		- addr &= PAGE_MASK;
2546		-
2547		- return cpu_buffer->commit_page->page == (void *)addr &&
2548		- rb_commit_index(cpu_buffer) == index;
2549		-}
2550		-
2551		-static __always_inline void
2552		-rb_update_write_stamp(struct ring_buffer_per_cpu *cpu_buffer,
2553		- struct ring_buffer_event *event)
2554		-{
2555		- u64 delta;
2556		-
2557		- /*
2558		- * The event first in the commit queue updates the
2559		- * time stamp.
2560		- */
2561		- if (rb_event_is_commit(cpu_buffer, event)) {
2562		- /*
2563		- * A commit event that is first on a page
2564		- * updates the write timestamp with the page stamp
2565		- */
2566		- if (!rb_event_index(event))
2567		- cpu_buffer->write_stamp =
2568		- cpu_buffer->commit_page->page->time_stamp;
2569		- else if (event->type_len == RINGBUF_TYPE_TIME_EXTEND) {
2570		- delta = ring_buffer_event_time_stamp(event);
2571		- cpu_buffer->write_stamp += delta;
2572		- } else if (event->type_len == RINGBUF_TYPE_TIME_STAMP) {
2573		- delta = ring_buffer_event_time_stamp(event);
2574		- cpu_buffer->write_stamp = delta;
2575		- } else
2576		- cpu_buffer->write_stamp += event->time_delta;
2577		- }
2578		-}
2579		-
2580	3061	static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
2581	3062	struct ring_buffer_event *event)
2582	3063	{
2583	3064	local_inc(&cpu_buffer->entries);
2584		- rb_update_write_stamp(cpu_buffer, event);
2585	3065	rb_end_commit(cpu_buffer);
2586	3066	}
2587	3067
2588	3068	static __always_inline void
2589		-rb_wakeups(struct ring_buffer buffer, struct ring_buffer_per_cpu cpu_buffer)
	3069	+rb_wakeups(struct trace_buffer buffer, struct ring_buffer_per_cpu cpu_buffer)
2590	3070	{
2591		- bool pagebusy;
2592		-
2593	3071	if (buffer->irq_work.waiters_pending) {
2594	3072	buffer->irq_work.waiters_pending = false;
2595	3073	/* irq_work_queue() supplies it's own memory barriers */
..	..	@@ -2602,14 +3080,24 @@
2602	3080	irq_work_queue(&cpu_buffer->irq_work.work);
2603	3081	}
2604	3082
2605		- pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
	3083	+ if (cpu_buffer->last_pages_touch == local_read(&cpu_buffer->pages_touched))
	3084	+ return;
2606	3085
2607		- if (!pagebusy && cpu_buffer->irq_work.full_waiters_pending) {
2608		- cpu_buffer->irq_work.wakeup_full = true;
2609		- cpu_buffer->irq_work.full_waiters_pending = false;
2610		- /* irq_work_queue() supplies it's own memory barriers */
2611		- irq_work_queue(&cpu_buffer->irq_work.work);
2612		- }
	3086	+ if (cpu_buffer->reader_page == cpu_buffer->commit_page)
	3087	+ return;
	3088	+
	3089	+ if (!cpu_buffer->irq_work.full_waiters_pending)
	3090	+ return;
	3091	+
	3092	+ cpu_buffer->last_pages_touch = local_read(&cpu_buffer->pages_touched);
	3093	+
	3094	+ if (!full_hit(buffer, cpu_buffer->cpu, cpu_buffer->shortest_full))
	3095	+ return;
	3096	+
	3097	+ cpu_buffer->irq_work.wakeup_full = true;
	3098	+ cpu_buffer->irq_work.full_waiters_pending = false;
	3099	+ /* irq_work_queue() supplies it's own memory barriers */
	3100	+ irq_work_queue(&cpu_buffer->irq_work.work);
2613	3101	}
2614	3102
2615	3103	/*
..	..	@@ -2727,7 +3215,7 @@
2727	3215	* Call this function before calling another ring_buffer_lock_reserve() and
2728	3216	* call ring_buffer_nest_end() after the nested ring_buffer_unlock_commit().
2729	3217	*/
2730		-void ring_buffer_nest_start(struct ring_buffer *buffer)
	3218	+void ring_buffer_nest_start(struct trace_buffer *buffer)
2731	3219	{
2732	3220	struct ring_buffer_per_cpu *cpu_buffer;
2733	3221	int cpu;
..	..	@@ -2747,7 +3235,7 @@
2747	3235	* Must be called after ring_buffer_nest_start() and after the
2748	3236	* ring_buffer_unlock_commit().
2749	3237	*/
2750		-void ring_buffer_nest_end(struct ring_buffer *buffer)
	3238	+void ring_buffer_nest_end(struct trace_buffer *buffer)
2751	3239	{
2752	3240	struct ring_buffer_per_cpu *cpu_buffer;
2753	3241	int cpu;
..	..	@@ -2769,7 +3257,7 @@
2769	3257	*
2770	3258	* Must be paired with ring_buffer_lock_reserve.
2771	3259	*/
2772		-int ring_buffer_unlock_commit(struct ring_buffer *buffer,
	3260	+int ring_buffer_unlock_commit(struct trace_buffer *buffer,
2773	3261	struct ring_buffer_event *event)
2774	3262	{
2775	3263	struct ring_buffer_per_cpu *cpu_buffer;
..	..	@@ -2789,57 +3277,135 @@
2789	3277	}
2790	3278	EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);
2791	3279
2792		-static noinline void
2793		-rb_handle_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
2794		- struct rb_event_info *info)
2795		-{
2796		- WARN_ONCE(info->delta > (1ULL << 59),
2797		- KERN_WARNING "Delta way too big! %llu ts=%llu write stamp = %llu\n%s",
2798		- (unsigned long long)info->delta,
2799		- (unsigned long long)info->ts,
2800		- (unsigned long long)cpu_buffer->write_stamp,
2801		- sched_clock_stable() ? "" :
2802		- "If you just came from a suspend/resume,\n"
2803		- "please switch to the trace global clock:\n"
2804		- " echo global > /sys/kernel/debug/tracing/trace_clock\n"
2805		- "or add trace_clock=global to the kernel command line\n");
2806		- info->add_timestamp = 1;
2807		-}
2808		-
2809	3280	static struct ring_buffer_event *
2810	3281	__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
2811	3282	struct rb_event_info *info)
2812	3283	{
2813	3284	struct ring_buffer_event *event;
2814	3285	struct buffer_page *tail_page;
2815		- unsigned long tail, write;
2816		-
2817		- /*
2818		- * If the time delta since the last event is too big to
2819		- * hold in the time field of the event, then we append a
2820		- * TIME EXTEND event ahead of the data event.
2821		- */
2822		- if (unlikely(info->add_timestamp))
2823		- info->length += RB_LEN_TIME_EXTEND;
	3286	+ unsigned long tail, write, w;
	3287	+ bool a_ok;
	3288	+ bool b_ok;
2824	3289
2825	3290	/* Don't let the compiler play games with cpu_buffer->tail_page */
2826	3291	tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page);
2827		- write = local_add_return(info->length, &tail_page->write);
	3292	+
	3293	+ /A/ w = local_read(&tail_page->write) & RB_WRITE_MASK;
	3294	+ barrier();
	3295	+ b_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before);
	3296	+ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);
	3297	+ barrier();
	3298	+ info->ts = rb_time_stamp(cpu_buffer->buffer);
	3299	+
	3300	+ if ((info->add_timestamp & RB_ADD_STAMP_ABSOLUTE)) {
	3301	+ info->delta = info->ts;
	3302	+ } else {
	3303	+ /*
	3304	+ * If interrupting an event time update, we may need an
	3305	+ * absolute timestamp.
	3306	+ * Don't bother if this is the start of a new page (w == 0).
	3307	+ */
	3308	+ if (unlikely(!a_ok \|\| !b_ok \|\| (info->before != info->after && w))) {
	3309	+ info->add_timestamp \|= RB_ADD_STAMP_FORCE \| RB_ADD_STAMP_EXTEND;
	3310	+ info->length += RB_LEN_TIME_EXTEND;
	3311	+ } else {
	3312	+ info->delta = info->ts - info->after;
	3313	+ if (unlikely(test_time_stamp(info->delta))) {
	3314	+ info->add_timestamp \|= RB_ADD_STAMP_EXTEND;
	3315	+ info->length += RB_LEN_TIME_EXTEND;
	3316	+ }
	3317	+ }
	3318	+ }
	3319	+
	3320	+ /B/ rb_time_set(&cpu_buffer->before_stamp, info->ts);
	3321	+
	3322	+ /C/ write = local_add_return(info->length, &tail_page->write);
2828	3323
2829	3324	/* set write to only the index of the write */
2830	3325	write &= RB_WRITE_MASK;
	3326	+
2831	3327	tail = write - info->length;
	3328	+
	3329	+ /* See if we shot pass the end of this buffer page */
	3330	+ if (unlikely(write > BUF_PAGE_SIZE)) {
	3331	+ /* before and after may now different, fix it up*/
	3332	+ b_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before);
	3333	+ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);
	3334	+ if (a_ok && b_ok && info->before != info->after)
	3335	+ (void)rb_time_cmpxchg(&cpu_buffer->before_stamp,
	3336	+ info->before, info->after);
	3337	+ return rb_move_tail(cpu_buffer, tail, info);
	3338	+ }
	3339	+
	3340	+ if (likely(tail == w)) {
	3341	+ u64 save_before;
	3342	+ bool s_ok;
	3343	+
	3344	+ /* Nothing interrupted us between A and C */
	3345	+ /D/ rb_time_set(&cpu_buffer->write_stamp, info->ts);
	3346	+ barrier();
	3347	+ /E/ s_ok = rb_time_read(&cpu_buffer->before_stamp, &save_before);
	3348	+ RB_WARN_ON(cpu_buffer, !s_ok);
	3349	+ if (likely(!(info->add_timestamp &
	3350	+ (RB_ADD_STAMP_FORCE \| RB_ADD_STAMP_ABSOLUTE))))
	3351	+ /* This did not interrupt any time update */
	3352	+ info->delta = info->ts - info->after;
	3353	+ else
	3354	+ /* Just use full timestamp for inerrupting event */
	3355	+ info->delta = info->ts;
	3356	+ barrier();
	3357	+ if (unlikely(info->ts != save_before)) {
	3358	+ /* SLOW PATH - Interrupted between C and E */
	3359	+
	3360	+ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);
	3361	+ RB_WARN_ON(cpu_buffer, !a_ok);
	3362	+
	3363	+ /* Write stamp must only go forward */
	3364	+ if (save_before > info->after) {
	3365	+ /*
	3366	+ * We do not care about the result, only that
	3367	+ * it gets updated atomically.
	3368	+ */
	3369	+ (void)rb_time_cmpxchg(&cpu_buffer->write_stamp,
	3370	+ info->after, save_before);
	3371	+ }
	3372	+ }
	3373	+ } else {
	3374	+ u64 ts;
	3375	+ /* SLOW PATH - Interrupted between A and C */
	3376	+ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);
	3377	+ /* Was interrupted before here, write_stamp must be valid */
	3378	+ RB_WARN_ON(cpu_buffer, !a_ok);
	3379	+ ts = rb_time_stamp(cpu_buffer->buffer);
	3380	+ barrier();
	3381	+ /E/ if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) &&
	3382	+ info->after < ts &&
	3383	+ rb_time_cmpxchg(&cpu_buffer->write_stamp,
	3384	+ info->after, ts)) {
	3385	+ /* Nothing came after this event between C and E */
	3386	+ info->delta = ts - info->after;
	3387	+ info->ts = ts;
	3388	+ } else {
	3389	+ /*
	3390	+ * Interrupted beween C and E:
	3391	+ * Lost the previous events time stamp. Just set the
	3392	+ * delta to zero, and this will be the same time as
	3393	+ * the event this event interrupted. And the events that
	3394	+ * came after this will still be correct (as they would
	3395	+ * have built their delta on the previous event.
	3396	+ */
	3397	+ info->delta = 0;
	3398	+ }
	3399	+ info->add_timestamp &= ~RB_ADD_STAMP_FORCE;
	3400	+ }
2832	3401
2833	3402	/*
2834	3403	* If this is the first commit on the page, then it has the same
2835	3404	* timestamp as the page itself.
2836	3405	*/
2837		- if (!tail && !ring_buffer_time_stamp_abs(cpu_buffer->buffer))
	3406	+ if (unlikely(!tail && !(info->add_timestamp &
	3407	+ (RB_ADD_STAMP_FORCE \| RB_ADD_STAMP_ABSOLUTE))))
2838	3408	info->delta = 0;
2839		-
2840		- /* See if we shot pass the end of this buffer page */
2841		- if (unlikely(write > BUF_PAGE_SIZE))
2842		- return rb_move_tail(cpu_buffer, tail, info);
2843	3409
2844	3410	/* We reserved something on the buffer */
2845	3411
..	..	@@ -2852,7 +3418,7 @@
2852	3418	* If this is the first commit on the page, then update
2853	3419	* its timestamp.
2854	3420	*/
2855		- if (!tail)
	3421	+ if (unlikely(!tail))
2856	3422	tail_page->page->time_stamp = info->ts;
2857	3423
2858	3424	/* account for these added bytes */
..	..	@@ -2862,16 +3428,17 @@
2862	3428	}
2863	3429
2864	3430	static __always_inline struct ring_buffer_event *
2865		-rb_reserve_next_event(struct ring_buffer *buffer,
	3431	+rb_reserve_next_event(struct trace_buffer *buffer,
2866	3432	struct ring_buffer_per_cpu *cpu_buffer,
2867	3433	unsigned long length)
2868	3434	{
2869	3435	struct ring_buffer_event *event;
2870	3436	struct rb_event_info info;
2871	3437	int nr_loops = 0;
2872		- u64 diff;
	3438	+ int add_ts_default;
2873	3439
2874	3440	rb_start_commit(cpu_buffer);
	3441	+ /* The commit page can not change after this */
2875	3442
2876	3443	#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2877	3444	/*
..	..	@@ -2889,8 +3456,16 @@
2889	3456	#endif
2890	3457
2891	3458	info.length = rb_calculate_event_length(length);
	3459	+
	3460	+ if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {
	3461	+ add_ts_default = RB_ADD_STAMP_ABSOLUTE;
	3462	+ info.length += RB_LEN_TIME_EXTEND;
	3463	+ } else {
	3464	+ add_ts_default = RB_ADD_STAMP_NONE;
	3465	+ }
	3466	+
2892	3467	again:
2893		- info.add_timestamp = 0;
	3468	+ info.add_timestamp = add_ts_default;
2894	3469	info.delta = 0;
2895	3470
2896	3471	/*
..	..	@@ -2905,35 +3480,16 @@
2905	3480	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
2906	3481	goto out_fail;
2907	3482
2908		- info.ts = rb_time_stamp(cpu_buffer->buffer);
2909		- diff = info.ts - cpu_buffer->write_stamp;
2910		-
2911		- /* make sure this diff is calculated here */
2912		- barrier();
2913		-
2914		- if (ring_buffer_time_stamp_abs(buffer)) {
2915		- info.delta = info.ts;
2916		- rb_handle_timestamp(cpu_buffer, &info);
2917		- } else /* Did the write stamp get updated already? */
2918		- if (likely(info.ts >= cpu_buffer->write_stamp)) {
2919		- info.delta = diff;
2920		- if (unlikely(test_time_stamp(info.delta)))
2921		- rb_handle_timestamp(cpu_buffer, &info);
2922		- }
2923		-
2924	3483	event = __rb_reserve_next(cpu_buffer, &info);
2925	3484
2926	3485	if (unlikely(PTR_ERR(event) == -EAGAIN)) {
2927		- if (info.add_timestamp)
	3486	+ if (info.add_timestamp & (RB_ADD_STAMP_FORCE \| RB_ADD_STAMP_EXTEND))
2928	3487	info.length -= RB_LEN_TIME_EXTEND;
2929	3488	goto again;
2930	3489	}
2931	3490
2932		- if (!event)
2933		- goto out_fail;
2934		-
2935		- return event;
2936		-
	3491	+ if (likely(event))
	3492	+ return event;
2937	3493	out_fail:
2938	3494	rb_end_commit(cpu_buffer);
2939	3495	return NULL;
..	..	@@ -2955,7 +3511,7 @@
2955	3511	* If NULL is returned, then nothing has been allocated or locked.
2956	3512	*/
2957	3513	struct ring_buffer_event *
2958		-ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
	3514	+ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length)
2959	3515	{
2960	3516	struct ring_buffer_per_cpu *cpu_buffer;
2961	3517	struct ring_buffer_event *event;
..	..	@@ -3056,7 +3612,7 @@
3056	3612	* If this function is called, do not call ring_buffer_unlock_commit on
3057	3613	* the event.
3058	3614	*/
3059		-void ring_buffer_discard_commit(struct ring_buffer *buffer,
	3615	+void ring_buffer_discard_commit(struct trace_buffer *buffer,
3060	3616	struct ring_buffer_event *event)
3061	3617	{
3062	3618	struct ring_buffer_per_cpu *cpu_buffer;
..	..	@@ -3079,11 +3635,6 @@
3079	3635	if (rb_try_to_discard(cpu_buffer, event))
3080	3636	goto out;
3081	3637
3082		- /*
3083		- * The commit is still visible by the reader, so we
3084		- * must still update the timestamp.
3085		- */
3086		- rb_update_write_stamp(cpu_buffer, event);
3087	3638	out:
3088	3639	rb_end_commit(cpu_buffer);
3089	3640
..	..	@@ -3107,7 +3658,7 @@
3107	3658	* Note, like ring_buffer_lock_reserve, the length is the length of the data
3108	3659	* and not the length of the event which would hold the header.
3109	3660	*/
3110		-int ring_buffer_write(struct ring_buffer *buffer,
	3661	+int ring_buffer_write(struct trace_buffer *buffer,
3111	3662	unsigned long length,
3112	3663	void *data)
3113	3664	{
..	..	@@ -3205,9 +3756,9 @@
3205	3756	* This prevents all writes to the buffer. Any attempt to write
3206	3757	* to the buffer after this will fail and return NULL.
3207	3758	*
3208		- * The caller should call synchronize_sched() after this.
	3759	+ * The caller should call synchronize_rcu() after this.
3209	3760	*/
3210		-void ring_buffer_record_disable(struct ring_buffer *buffer)
	3761	+void ring_buffer_record_disable(struct trace_buffer *buffer)
3211	3762	{
3212	3763	atomic_inc(&buffer->record_disabled);
3213	3764	}
..	..	@@ -3220,7 +3771,7 @@
3220	3771	* Note, multiple disables will need the same number of enables
3221	3772	* to truly enable the writing (much like preempt_disable).
3222	3773	*/
3223		-void ring_buffer_record_enable(struct ring_buffer *buffer)
	3774	+void ring_buffer_record_enable(struct trace_buffer *buffer)
3224	3775	{
3225	3776	atomic_dec(&buffer->record_disabled);
3226	3777	}
..	..	@@ -3237,7 +3788,7 @@
3237	3788	* it works like an on/off switch, where as the disable() version
3238	3789	* must be paired with a enable().
3239	3790	*/
3240		-void ring_buffer_record_off(struct ring_buffer *buffer)
	3791	+void ring_buffer_record_off(struct trace_buffer *buffer)
3241	3792	{
3242	3793	unsigned int rd;
3243	3794	unsigned int new_rd;
..	..	@@ -3260,7 +3811,7 @@
3260	3811	* it works like an on/off switch, where as the enable() version
3261	3812	* must be paired with a disable().
3262	3813	*/
3263		-void ring_buffer_record_on(struct ring_buffer *buffer)
	3814	+void ring_buffer_record_on(struct trace_buffer *buffer)
3264	3815	{
3265	3816	unsigned int rd;
3266	3817	unsigned int new_rd;
..	..	@@ -3278,7 +3829,7 @@
3278	3829	*
3279	3830	* Returns true if the ring buffer is in a state that it accepts writes.
3280	3831	*/
3281		-bool ring_buffer_record_is_on(struct ring_buffer *buffer)
	3832	+bool ring_buffer_record_is_on(struct trace_buffer *buffer)
3282	3833	{
3283	3834	return !atomic_read(&buffer->record_disabled);
3284	3835	}
..	..	@@ -3294,7 +3845,7 @@
3294	3845	* ring_buffer_record_disable(), as that is a temporary disabling of
3295	3846	* the ring buffer.
3296	3847	*/
3297		-bool ring_buffer_record_is_set_on(struct ring_buffer *buffer)
	3848	+bool ring_buffer_record_is_set_on(struct trace_buffer *buffer)
3298	3849	{
3299	3850	return !(atomic_read(&buffer->record_disabled) & RB_BUFFER_OFF);
3300	3851	}
..	..	@@ -3307,9 +3858,9 @@
3307	3858	* This prevents all writes to the buffer. Any attempt to write
3308	3859	* to the buffer after this will fail and return NULL.
3309	3860	*
3310		- * The caller should call synchronize_sched() after this.
	3861	+ * The caller should call synchronize_rcu() after this.
3311	3862	*/
3312		-void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
	3863	+void ring_buffer_record_disable_cpu(struct trace_buffer *buffer, int cpu)
3313	3864	{
3314	3865	struct ring_buffer_per_cpu *cpu_buffer;
3315	3866
..	..	@@ -3329,7 +3880,7 @@
3329	3880	* Note, multiple disables will need the same number of enables
3330	3881	* to truly enable the writing (much like preempt_disable).
3331	3882	*/
3332		-void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
	3883	+void ring_buffer_record_enable_cpu(struct trace_buffer *buffer, int cpu)
3333	3884	{
3334	3885	struct ring_buffer_per_cpu *cpu_buffer;
3335	3886
..	..	@@ -3359,7 +3910,7 @@
3359	3910	* @buffer: The ring buffer
3360	3911	* @cpu: The per CPU buffer to read from.
3361	3912	*/
3362		-u64 ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu)
	3913	+u64 ring_buffer_oldest_event_ts(struct trace_buffer *buffer, int cpu)
3363	3914	{
3364	3915	unsigned long flags;
3365	3916	struct ring_buffer_per_cpu *cpu_buffer;
..	..	@@ -3392,7 +3943,7 @@
3392	3943	* @buffer: The ring buffer
3393	3944	* @cpu: The per CPU buffer to read from.
3394	3945	*/
3395		-unsigned long ring_buffer_bytes_cpu(struct ring_buffer *buffer, int cpu)
	3946	+unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu)
3396	3947	{
3397	3948	struct ring_buffer_per_cpu *cpu_buffer;
3398	3949	unsigned long ret;
..	..	@@ -3412,7 +3963,7 @@
3412	3963	* @buffer: The ring buffer
3413	3964	* @cpu: The per CPU buffer to get the entries from.
3414	3965	*/
3415		-unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
	3966	+unsigned long ring_buffer_entries_cpu(struct trace_buffer *buffer, int cpu)
3416	3967	{
3417	3968	struct ring_buffer_per_cpu *cpu_buffer;
3418	3969
..	..	@@ -3431,7 +3982,7 @@
3431	3982	* @buffer: The ring buffer
3432	3983	* @cpu: The per CPU buffer to get the number of overruns from
3433	3984	*/
3434		-unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
	3985	+unsigned long ring_buffer_overrun_cpu(struct trace_buffer *buffer, int cpu)
3435	3986	{
3436	3987	struct ring_buffer_per_cpu *cpu_buffer;
3437	3988	unsigned long ret;
..	..	@@ -3454,7 +4005,7 @@
3454	4005	* @cpu: The per CPU buffer to get the number of overruns from
3455	4006	*/
3456	4007	unsigned long
3457		-ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)
	4008	+ring_buffer_commit_overrun_cpu(struct trace_buffer *buffer, int cpu)
3458	4009	{
3459	4010	struct ring_buffer_per_cpu *cpu_buffer;
3460	4011	unsigned long ret;
..	..	@@ -3476,7 +4027,7 @@
3476	4027	* @cpu: The per CPU buffer to get the number of overruns from
3477	4028	*/
3478	4029	unsigned long
3479		-ring_buffer_dropped_events_cpu(struct ring_buffer *buffer, int cpu)
	4030	+ring_buffer_dropped_events_cpu(struct trace_buffer *buffer, int cpu)
3480	4031	{
3481	4032	struct ring_buffer_per_cpu *cpu_buffer;
3482	4033	unsigned long ret;
..	..	@@ -3497,7 +4048,7 @@
3497	4048	* @cpu: The per CPU buffer to get the number of events read
3498	4049	*/
3499	4050	unsigned long
3500		-ring_buffer_read_events_cpu(struct ring_buffer *buffer, int cpu)
	4051	+ring_buffer_read_events_cpu(struct trace_buffer *buffer, int cpu)
3501	4052	{
3502	4053	struct ring_buffer_per_cpu *cpu_buffer;
3503	4054
..	..	@@ -3516,7 +4067,7 @@
3516	4067	* Returns the total number of entries in the ring buffer
3517	4068	* (all CPU entries)
3518	4069	*/
3519		-unsigned long ring_buffer_entries(struct ring_buffer *buffer)
	4070	+unsigned long ring_buffer_entries(struct trace_buffer *buffer)
3520	4071	{
3521	4072	struct ring_buffer_per_cpu *cpu_buffer;
3522	4073	unsigned long entries = 0;
..	..	@@ -3539,7 +4090,7 @@
3539	4090	* Returns the total number of overruns in the ring buffer
3540	4091	* (all CPU entries)
3541	4092	*/
3542		-unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
	4093	+unsigned long ring_buffer_overruns(struct trace_buffer *buffer)
3543	4094	{
3544	4095	struct ring_buffer_per_cpu *cpu_buffer;
3545	4096	unsigned long overruns = 0;
..	..	@@ -3562,14 +4113,18 @@
3562	4113	/* Iterator usage is expected to have record disabled */
3563	4114	iter->head_page = cpu_buffer->reader_page;
3564	4115	iter->head = cpu_buffer->reader_page->read;
	4116	+ iter->next_event = iter->head;
3565	4117
3566	4118	iter->cache_reader_page = iter->head_page;
3567	4119	iter->cache_read = cpu_buffer->read;
3568	4120
3569		- if (iter->head)
	4121	+ if (iter->head) {
3570	4122	iter->read_stamp = cpu_buffer->read_stamp;
3571		- else
	4123	+ iter->page_stamp = cpu_buffer->reader_page->page->time_stamp;
	4124	+ } else {
3572	4125	iter->read_stamp = iter->head_page->page->time_stamp;
	4126	+ iter->page_stamp = iter->read_stamp;
	4127	+ }
3573	4128	}
3574	4129
3575	4130	/**
..	..	@@ -3605,17 +4160,38 @@
3605	4160	struct buffer_page *reader;
3606	4161	struct buffer_page *head_page;
3607	4162	struct buffer_page *commit_page;
	4163	+ struct buffer_page *curr_commit_page;
3608	4164	unsigned commit;
	4165	+ u64 curr_commit_ts;
	4166	+ u64 commit_ts;
3609	4167
3610	4168	cpu_buffer = iter->cpu_buffer;
3611		-
3612		- /* Remember, trace recording is off when iterator is in use */
3613	4169	reader = cpu_buffer->reader_page;
3614	4170	head_page = cpu_buffer->head_page;
3615	4171	commit_page = cpu_buffer->commit_page;
3616		- commit = rb_page_commit(commit_page);
	4172	+ commit_ts = commit_page->page->time_stamp;
3617	4173
3618		- return ((iter->head_page == commit_page && iter->head == commit) \|\|
	4174	+ /*
	4175	+ * When the writer goes across pages, it issues a cmpxchg which
	4176	+ * is a mb(), which will synchronize with the rmb here.
	4177	+ * (see rb_tail_page_update())
	4178	+ */
	4179	+ smp_rmb();
	4180	+ commit = rb_page_commit(commit_page);
	4181	+ /* We want to make sure that the commit page doesn't change */
	4182	+ smp_rmb();
	4183	+
	4184	+ /* Make sure commit page didn't change */
	4185	+ curr_commit_page = READ_ONCE(cpu_buffer->commit_page);
	4186	+ curr_commit_ts = READ_ONCE(curr_commit_page->page->time_stamp);
	4187	+
	4188	+ /* If the commit page changed, then there's more data */
	4189	+ if (curr_commit_page != commit_page \|\|
	4190	+ curr_commit_ts != commit_ts)
	4191	+ return 0;
	4192	+
	4193	+ /* Still racy, as it may return a false positive, but that's OK */
	4194	+ return ((iter->head_page == commit_page && iter->head >= commit) \|\|
3619	4195	(iter->head_page == reader && commit_page == head_page &&
3620	4196	head_page->read == commit &&
3621	4197	iter->head == rb_page_commit(cpu_buffer->reader_page)));
..	..	@@ -3647,7 +4223,7 @@
3647	4223	return;
3648	4224
3649	4225	default:
3650		- BUG();
	4226	+ RB_WARN_ON(cpu_buffer, 1);
3651	4227	}
3652	4228	return;
3653	4229	}
..	..	@@ -3677,7 +4253,7 @@
3677	4253	return;
3678	4254
3679	4255	default:
3680		- BUG();
	4256	+ RB_WARN_ON(iter->cpu_buffer, 1);
3681	4257	}
3682	4258	return;
3683	4259	}
..	..	@@ -3786,12 +4362,14 @@
3786	4362	goto spin;
3787	4363
3788	4364	/*
3789		- * Yeah! We succeeded in replacing the page.
	4365	+ * Yay! We succeeded in replacing the page.
3790	4366	*
3791	4367	* Now make the new head point back to the reader page.
3792	4368	*/
3793	4369	rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;
3794	4370	rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
	4371	+
	4372	+ local_inc(&cpu_buffer->pages_read);
3795	4373
3796	4374	/* Finally update the reader page to the new head */
3797	4375	cpu_buffer->reader_page = reader;
..	..	@@ -3811,6 +4389,33 @@
3811	4389
3812	4390	arch_spin_unlock(&cpu_buffer->lock);
3813	4391	local_irq_restore(flags);
	4392	+
	4393	+ /*
	4394	+ * The writer has preempt disable, wait for it. But not forever
	4395	+ * Although, 1 second is pretty much "forever"
	4396	+ */
	4397	+#define USECS_WAIT 1000000
	4398	+ for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {
	4399	+ /* If the write is past the end of page, a writer is still updating it */
	4400	+ if (likely(!reader \|\| rb_page_write(reader) <= BUF_PAGE_SIZE))
	4401	+ break;
	4402	+
	4403	+ udelay(1);
	4404	+
	4405	+ /* Get the latest version of the reader write value */
	4406	+ smp_rmb();
	4407	+ }
	4408	+
	4409	+ /* The writer is not moving forward? Something is wrong */
	4410	+ if (RB_WARN_ON(cpu_buffer, nr_loops == USECS_WAIT))
	4411	+ reader = NULL;
	4412	+
	4413	+ /*
	4414	+ * Make sure we see any padding after the write update
	4415	+ * (see rb_reset_tail())
	4416	+ */
	4417	+ smp_rmb();
	4418	+
3814	4419
3815	4420	return reader;
3816	4421	}
..	..	@@ -3841,15 +4446,22 @@
3841	4446	static void rb_advance_iter(struct ring_buffer_iter *iter)
3842	4447	{
3843	4448	struct ring_buffer_per_cpu *cpu_buffer;
3844		- struct ring_buffer_event *event;
3845		- unsigned length;
3846	4449
3847	4450	cpu_buffer = iter->cpu_buffer;
	4451	+
	4452	+ /* If head == next_event then we need to jump to the next event */
	4453	+ if (iter->head == iter->next_event) {
	4454	+ /* If the event gets overwritten again, there's nothing to do */
	4455	+ if (rb_iter_head_event(iter) == NULL)
	4456	+ return;
	4457	+ }
	4458	+
	4459	+ iter->head = iter->next_event;
3848	4460
3849	4461	/*
3850	4462	* Check if we are at the end of the buffer.
3851	4463	*/
3852		- if (iter->head >= rb_page_size(iter->head_page)) {
	4464	+ if (iter->next_event >= rb_page_size(iter->head_page)) {
3853	4465	/* discarded commits can make the page empty */
3854	4466	if (iter->head_page == cpu_buffer->commit_page)
3855	4467	return;
..	..	@@ -3857,27 +4469,7 @@
3857	4469	return;
3858	4470	}
3859	4471
3860		- event = rb_iter_head_event(iter);
3861		-
3862		- length = rb_event_length(event);
3863		-
3864		- /*
3865		- * This should not be called to advance the header if we are
3866		- * at the tail of the buffer.
3867		- */
3868		- if (RB_WARN_ON(cpu_buffer,
3869		- (iter->head_page == cpu_buffer->commit_page) &&
3870		- (iter->head + length > rb_commit_index(cpu_buffer))))
3871		- return;
3872		-
3873		- rb_update_iter_read_stamp(iter, event);
3874		-
3875		- iter->head += length;
3876		-
3877		- /* check for end of page padding */
3878		- if ((iter->head >= rb_page_size(iter->head_page)) &&
3879		- (iter->head_page != cpu_buffer->commit_page))
3880		- rb_inc_iter(iter);
	4472	+ rb_update_iter_read_stamp(iter, iter->event);
3881	4473	}
3882	4474
3883	4475	static int rb_lost_events(struct ring_buffer_per_cpu *cpu_buffer)
..	..	@@ -3951,7 +4543,7 @@
3951	4543	return event;
3952	4544
3953	4545	default:
3954		- BUG();
	4546	+ RB_WARN_ON(cpu_buffer, 1);
3955	4547	}
3956	4548
3957	4549	return NULL;
..	..	@@ -3961,7 +4553,7 @@
3961	4553	static struct ring_buffer_event *
3962	4554	rb_iter_peek(struct ring_buffer_iter iter, u64 ts)
3963	4555	{
3964		- struct ring_buffer *buffer;
	4556	+ struct trace_buffer *buffer;
3965	4557	struct ring_buffer_per_cpu *cpu_buffer;
3966	4558	struct ring_buffer_event *event;
3967	4559	int nr_loops = 0;
..	..	@@ -3986,14 +4578,13 @@
3986	4578	return NULL;
3987	4579
3988	4580	/*
3989		- * We repeat when a time extend is encountered or we hit
3990		- * the end of the page. Since the time extend is always attached
3991		- * to a data event, we should never loop more than three times.
3992		- * Once for going to next page, once on time extend, and
3993		- * finally once to get the event.
3994		- * (We never hit the following condition more than thrice).
	4581	+ * As the writer can mess with what the iterator is trying
	4582	+ * to read, just give up if we fail to get an event after
	4583	+ * three tries. The iterator is not as reliable when reading
	4584	+ * the ring buffer with an active write as the consumer is.
	4585	+ * Do not warn if the three failures is reached.
3995	4586	*/
3996		- if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3))
	4587	+ if (++nr_loops > 3)
3997	4588	return NULL;
3998	4589
3999	4590	if (rb_per_cpu_empty(cpu_buffer))
..	..	@@ -4005,6 +4596,8 @@
4005	4596	}
4006	4597
4007	4598	event = rb_iter_head_event(iter);
	4599	+ if (!event)
	4600	+ goto again;
4008	4601
4009	4602	switch (event->type_len) {
4010	4603	case RINGBUF_TYPE_PADDING:
..	..	@@ -4039,7 +4632,7 @@
4039	4632	return event;
4040	4633
4041	4634	default:
4042		- BUG();
	4635	+ RB_WARN_ON(cpu_buffer, 1);
4043	4636	}
4044	4637
4045	4638	return NULL;
..	..	@@ -4089,7 +4682,7 @@
4089	4682	* not consume the data.
4090	4683	*/
4091	4684	struct ring_buffer_event *
4092		-ring_buffer_peek(struct ring_buffer buffer, int cpu, u64 ts,
	4685	+ring_buffer_peek(struct trace_buffer buffer, int cpu, u64 ts,
4093	4686	unsigned long *lost_events)
4094	4687	{
4095	4688	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
..	..	@@ -4114,6 +4707,20 @@
4114	4707
4115	4708	return event;
4116	4709	}
	4710	+
	4711	+/** ring_buffer_iter_dropped - report if there are dropped events
	4712	+ * @iter: The ring buffer iterator
	4713	+ *
	4714	+ * Returns true if there was dropped events since the last peek.
	4715	+ */
	4716	+bool ring_buffer_iter_dropped(struct ring_buffer_iter *iter)
	4717	+{
	4718	+ bool ret = iter->missed_events != 0;
	4719	+
	4720	+ iter->missed_events = 0;
	4721	+ return ret;
	4722	+}
	4723	+EXPORT_SYMBOL_GPL(ring_buffer_iter_dropped);
4117	4724
4118	4725	/**
4119	4726	* ring_buffer_iter_peek - peek at the next event to be read
..	..	@@ -4153,7 +4760,7 @@
4153	4760	* and eventually empty the ring buffer if the producer is slower.
4154	4761	*/
4155	4762	struct ring_buffer_event *
4156		-ring_buffer_consume(struct ring_buffer buffer, int cpu, u64 ts,
	4763	+ring_buffer_consume(struct trace_buffer buffer, int cpu, u64 ts,
4157	4764	unsigned long *lost_events)
4158	4765	{
4159	4766	struct ring_buffer_per_cpu *cpu_buffer;
..	..	@@ -4213,7 +4820,7 @@
4213	4820	* This overall must be paired with ring_buffer_read_finish.
4214	4821	*/
4215	4822	struct ring_buffer_iter *
4216		-ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu, gfp_t flags)
	4823	+ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags)
4217	4824	{
4218	4825	struct ring_buffer_per_cpu *cpu_buffer;
4219	4826	struct ring_buffer_iter *iter;
..	..	@@ -4221,16 +4828,21 @@
4221	4828	if (!cpumask_test_cpu(cpu, buffer->cpumask))
4222	4829	return NULL;
4223	4830
4224		- iter = kmalloc(sizeof(*iter), flags);
	4831	+ iter = kzalloc(sizeof(*iter), flags);
4225	4832	if (!iter)
4226	4833	return NULL;
	4834	+
	4835	+ iter->event = kmalloc(BUF_MAX_DATA_SIZE, flags);
	4836	+ if (!iter->event) {
	4837	+ kfree(iter);
	4838	+ return NULL;
	4839	+ }
4227	4840
4228	4841	cpu_buffer = buffer->buffers[cpu];
4229	4842
4230	4843	iter->cpu_buffer = cpu_buffer;
4231	4844
4232		- atomic_inc(&buffer->resize_disabled);
4233		- atomic_inc(&cpu_buffer->record_disabled);
	4845	+ atomic_inc(&cpu_buffer->resize_disabled);
4234	4846
4235	4847	return iter;
4236	4848	}
..	..	@@ -4246,7 +4858,7 @@
4246	4858	void
4247	4859	ring_buffer_read_prepare_sync(void)
4248	4860	{
4249		- synchronize_sched();
	4861	+ synchronize_rcu();
4250	4862	}
4251	4863	EXPORT_SYMBOL_GPL(ring_buffer_read_prepare_sync);
4252	4864
..	..	@@ -4303,48 +4915,38 @@
4303	4915	rb_check_pages(cpu_buffer);
4304	4916	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4305	4917
4306		- atomic_dec(&cpu_buffer->record_disabled);
4307		- atomic_dec(&cpu_buffer->buffer->resize_disabled);
	4918	+ atomic_dec(&cpu_buffer->resize_disabled);
	4919	+ kfree(iter->event);
4308	4920	kfree(iter);
4309	4921	}
4310	4922	EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
4311	4923
4312	4924	/**
4313		- * ring_buffer_read - read the next item in the ring buffer by the iterator
	4925	+ * ring_buffer_iter_advance - advance the iterator to the next location
4314	4926	* @iter: The ring buffer iterator
4315		- * @ts: The time stamp of the event read.
4316	4927	*
4317		- * This reads the next event in the ring buffer and increments the iterator.
	4928	+ * Move the location of the iterator such that the next read will
	4929	+ * be the next location of the iterator.
4318	4930	*/
4319		-struct ring_buffer_event *
4320		-ring_buffer_read(struct ring_buffer_iter iter, u64 ts)
	4931	+void ring_buffer_iter_advance(struct ring_buffer_iter *iter)
4321	4932	{
4322		- struct ring_buffer_event *event;
4323	4933	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
4324	4934	unsigned long flags;
4325	4935
4326	4936	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
4327		- again:
4328		- event = rb_iter_peek(iter, ts);
4329		- if (!event)
4330		- goto out;
4331		-
4332		- if (event->type_len == RINGBUF_TYPE_PADDING)
4333		- goto again;
4334	4937
4335	4938	rb_advance_iter(iter);
4336		- out:
4337		- raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4338	4939
4339		- return event;
	4940	+ raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4340	4941	}
4341		-EXPORT_SYMBOL_GPL(ring_buffer_read);
	4942	+EXPORT_SYMBOL_GPL(ring_buffer_iter_advance);
4342	4943
4343	4944	/**
4344	4945	* ring_buffer_size - return the size of the ring buffer (in bytes)
4345	4946	* @buffer: The ring buffer.
	4947	+ * @cpu: The CPU to get ring buffer size from.
4346	4948	*/
4347		-unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu)
	4949	+unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu)
4348	4950	{
4349	4951	/*
4350	4952	* Earlier, this method returned
..	..	@@ -4389,11 +4991,16 @@
4389	4991	local_set(&cpu_buffer->entries, 0);
4390	4992	local_set(&cpu_buffer->committing, 0);
4391	4993	local_set(&cpu_buffer->commits, 0);
	4994	+ local_set(&cpu_buffer->pages_touched, 0);
	4995	+ local_set(&cpu_buffer->pages_lost, 0);
	4996	+ local_set(&cpu_buffer->pages_read, 0);
	4997	+ cpu_buffer->last_pages_touch = 0;
	4998	+ cpu_buffer->shortest_full = 0;
4392	4999	cpu_buffer->read = 0;
4393	5000	cpu_buffer->read_bytes = 0;
4394	5001
4395		- cpu_buffer->write_stamp = 0;
4396		- cpu_buffer->read_stamp = 0;
	5002	+ rb_time_set(&cpu_buffer->write_stamp, 0);
	5003	+ rb_time_set(&cpu_buffer->before_stamp, 0);
4397	5004
4398	5005	cpu_buffer->lost_events = 0;
4399	5006	cpu_buffer->last_overrun = 0;
..	..	@@ -4401,26 +5008,10 @@
4401	5008	rb_head_page_activate(cpu_buffer);
4402	5009	}
4403	5010
4404		-/**
4405		- * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
4406		- * @buffer: The ring buffer to reset a per cpu buffer of
4407		- * @cpu: The CPU buffer to be reset
4408		- */
4409		-void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
	5011	+/* Must have disabled the cpu buffer then done a synchronize_rcu */
	5012	+static void reset_disabled_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
4410	5013	{
4411		- struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
4412	5014	unsigned long flags;
4413		-
4414		- if (!cpumask_test_cpu(cpu, buffer->cpumask))
4415		- return;
4416		- /* prevent another thread from changing buffer sizes */
4417		- mutex_lock(&buffer->mutex);
4418		-
4419		- atomic_inc(&buffer->resize_disabled);
4420		- atomic_inc(&cpu_buffer->record_disabled);
4421		-
4422		- /* Make sure all commits have finished */
4423		- synchronize_sched();
4424	5015
4425	5016	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
4426	5017
..	..	@@ -4435,24 +5026,105 @@
4435	5026
4436	5027	out:
4437	5028	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
	5029	+}
	5030	+
	5031	+/**
	5032	+ * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
	5033	+ * @buffer: The ring buffer to reset a per cpu buffer of
	5034	+ * @cpu: The CPU buffer to be reset
	5035	+ */
	5036	+void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu)
	5037	+{
	5038	+ struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	5039	+
	5040	+ if (!cpumask_test_cpu(cpu, buffer->cpumask))
	5041	+ return;
	5042	+
	5043	+ /* prevent another thread from changing buffer sizes */
	5044	+ mutex_lock(&buffer->mutex);
	5045	+
	5046	+ atomic_inc(&cpu_buffer->resize_disabled);
	5047	+ atomic_inc(&cpu_buffer->record_disabled);
	5048	+
	5049	+ /* Make sure all commits have finished */
	5050	+ synchronize_rcu();
	5051	+
	5052	+ reset_disabled_cpu_buffer(cpu_buffer);
4438	5053
4439	5054	atomic_dec(&cpu_buffer->record_disabled);
4440		- atomic_dec(&buffer->resize_disabled);
	5055	+ atomic_dec(&cpu_buffer->resize_disabled);
4441	5056
4442	5057	mutex_unlock(&buffer->mutex);
4443	5058	}
4444	5059	EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
4445	5060
4446	5061	/**
	5062	+ * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
	5063	+ * @buffer: The ring buffer to reset a per cpu buffer of
	5064	+ * @cpu: The CPU buffer to be reset
	5065	+ */
	5066	+void ring_buffer_reset_online_cpus(struct trace_buffer *buffer)
	5067	+{
	5068	+ struct ring_buffer_per_cpu *cpu_buffer;
	5069	+ int cpu;
	5070	+
	5071	+ /* prevent another thread from changing buffer sizes */
	5072	+ mutex_lock(&buffer->mutex);
	5073	+
	5074	+ for_each_online_buffer_cpu(buffer, cpu) {
	5075	+ cpu_buffer = buffer->buffers[cpu];
	5076	+
	5077	+ atomic_inc(&cpu_buffer->resize_disabled);
	5078	+ atomic_inc(&cpu_buffer->record_disabled);
	5079	+ }
	5080	+
	5081	+ /* Make sure all commits have finished */
	5082	+ synchronize_rcu();
	5083	+
	5084	+ for_each_online_buffer_cpu(buffer, cpu) {
	5085	+ cpu_buffer = buffer->buffers[cpu];
	5086	+
	5087	+ reset_disabled_cpu_buffer(cpu_buffer);
	5088	+
	5089	+ atomic_dec(&cpu_buffer->record_disabled);
	5090	+ atomic_dec(&cpu_buffer->resize_disabled);
	5091	+ }
	5092	+
	5093	+ mutex_unlock(&buffer->mutex);
	5094	+}
	5095	+
	5096	+/**
4447	5097	* ring_buffer_reset - reset a ring buffer
4448	5098	* @buffer: The ring buffer to reset all cpu buffers
4449	5099	*/
4450		-void ring_buffer_reset(struct ring_buffer *buffer)
	5100	+void ring_buffer_reset(struct trace_buffer *buffer)
4451	5101	{
	5102	+ struct ring_buffer_per_cpu *cpu_buffer;
4452	5103	int cpu;
4453	5104
4454		- for_each_buffer_cpu(buffer, cpu)
4455		- ring_buffer_reset_cpu(buffer, cpu);
	5105	+ /* prevent another thread from changing buffer sizes */
	5106	+ mutex_lock(&buffer->mutex);
	5107	+
	5108	+ for_each_buffer_cpu(buffer, cpu) {
	5109	+ cpu_buffer = buffer->buffers[cpu];
	5110	+
	5111	+ atomic_inc(&cpu_buffer->resize_disabled);
	5112	+ atomic_inc(&cpu_buffer->record_disabled);
	5113	+ }
	5114	+
	5115	+ /* Make sure all commits have finished */
	5116	+ synchronize_rcu();
	5117	+
	5118	+ for_each_buffer_cpu(buffer, cpu) {
	5119	+ cpu_buffer = buffer->buffers[cpu];
	5120	+
	5121	+ reset_disabled_cpu_buffer(cpu_buffer);
	5122	+
	5123	+ atomic_dec(&cpu_buffer->record_disabled);
	5124	+ atomic_dec(&cpu_buffer->resize_disabled);
	5125	+ }
	5126	+
	5127	+ mutex_unlock(&buffer->mutex);
4456	5128	}
4457	5129	EXPORT_SYMBOL_GPL(ring_buffer_reset);
4458	5130
..	..	@@ -4460,7 +5132,7 @@
4460	5132	* rind_buffer_empty - is the ring buffer empty?
4461	5133	* @buffer: The ring buffer to test
4462	5134	*/
4463		-bool ring_buffer_empty(struct ring_buffer *buffer)
	5135	+bool ring_buffer_empty(struct trace_buffer *buffer)
4464	5136	{
4465	5137	struct ring_buffer_per_cpu *cpu_buffer;
4466	5138	unsigned long flags;
..	..	@@ -4490,7 +5162,7 @@
4490	5162	* @buffer: The ring buffer
4491	5163	* @cpu: The CPU buffer to test
4492	5164	*/
4493		-bool ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
	5165	+bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu)
4494	5166	{
4495	5167	struct ring_buffer_per_cpu *cpu_buffer;
4496	5168	unsigned long flags;
..	..	@@ -4516,14 +5188,15 @@
4516	5188	* ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
4517	5189	* @buffer_a: One buffer to swap with
4518	5190	* @buffer_b: The other buffer to swap with
	5191	+ * @cpu: the CPU of the buffers to swap
4519	5192	*
4520	5193	* This function is useful for tracers that want to take a "snapshot"
4521	5194	* of a CPU buffer and has another back up buffer lying around.
4522	5195	* it is expected that the tracer handles the cpu buffer not being
4523	5196	* used at the moment.
4524	5197	*/
4525		-int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
4526		- struct ring_buffer *buffer_b, int cpu)
	5198	+int ring_buffer_swap_cpu(struct trace_buffer *buffer_a,
	5199	+ struct trace_buffer *buffer_b, int cpu)
4527	5200	{
4528	5201	struct ring_buffer_per_cpu *cpu_buffer_a;
4529	5202	struct ring_buffer_per_cpu *cpu_buffer_b;
..	..	@@ -4555,7 +5228,7 @@
4555	5228	goto out;
4556	5229
4557	5230	/*
4558		- * We can't do a synchronize_sched here because this
	5231	+ * We can't do a synchronize_rcu here because this
4559	5232	* function can be called in atomic context.
4560	5233	* Normally this will be called from the same CPU as cpu.
4561	5234	* If not it's up to the caller to protect this.
..	..	@@ -4602,7 +5275,7 @@
4602	5275	* Returns:
4603	5276	* The page allocated, or ERR_PTR
4604	5277	*/
4605		-void ring_buffer_alloc_read_page(struct ring_buffer buffer, int cpu)
	5278	+void ring_buffer_alloc_read_page(struct trace_buffer buffer, int cpu)
4606	5279	{
4607	5280	struct ring_buffer_per_cpu *cpu_buffer;
4608	5281	struct buffer_data_page *bpage = NULL;
..	..	@@ -4649,7 +5322,7 @@
4649	5322	*
4650	5323	* Free a page allocated from ring_buffer_alloc_read_page.
4651	5324	*/
4652		-void ring_buffer_free_read_page(struct ring_buffer buffer, int cpu, void data)
	5325	+void ring_buffer_free_read_page(struct trace_buffer buffer, int cpu, void data)
4653	5326	{
4654	5327	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
4655	5328	struct buffer_data_page *bpage = data;
..	..	@@ -4709,7 +5382,7 @@
4709	5382	* >=0 if data has been transferred, returns the offset of consumed data.
4710	5383	* <0 if no data has been transferred.
4711	5384	*/
4712		-int ring_buffer_read_page(struct ring_buffer *buffer,
	5385	+int ring_buffer_read_page(struct trace_buffer *buffer,
4713	5386	void **data_page, size_t len, int cpu, int full)
4714	5387	{
4715	5388	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
..	..	@@ -4770,7 +5443,15 @@
4770	5443	unsigned int pos = 0;
4771	5444	unsigned int size;
4772	5445
4773		- if (full)
	5446	+ /*
	5447	+ * If a full page is expected, this can still be returned
	5448	+ * if there's been a previous partial read and the
	5449	+ * rest of the page can be read and the commit page is off
	5450	+ * the reader page.
	5451	+ */
	5452	+ if (full &&
	5453	+ (!read \|\| (len < (commit - read)) \|\|
	5454	+ cpu_buffer->reader_page == cpu_buffer->commit_page))
4774	5455	goto out_unlock;
4775	5456
4776	5457	if (len > (commit - read))
..	..	@@ -4880,12 +5561,12 @@
4880	5561	*/
4881	5562	int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node)
4882	5563	{
4883		- struct ring_buffer *buffer;
	5564	+ struct trace_buffer *buffer;
4884	5565	long nr_pages_same;
4885	5566	int cpu_i;
4886	5567	unsigned long nr_pages;
4887	5568
4888		- buffer = container_of(node, struct ring_buffer, node);
	5569	+ buffer = container_of(node, struct trace_buffer, node);
4889	5570	if (cpumask_test_cpu(cpu, buffer->cpumask))
4890	5571	return 0;
4891	5572
..	..	@@ -4935,7 +5616,7 @@
4935	5616	static struct task_struct *rb_threads[NR_CPUS] __initdata;
4936	5617
4937	5618	struct rb_test_data {
4938		- struct ring_buffer *buffer;
	5619	+ struct trace_buffer *buffer;
4939	5620	unsigned long events;
4940	5621	unsigned long bytes_written;
4941	5622	unsigned long bytes_alloc;
..	..	@@ -4983,7 +5664,7 @@
4983	5664	cnt = data->cnt + (nested ? 27 : 0);
4984	5665
4985	5666	/* Multiply cnt by ~e, to make some unique increment */
4986		- size = (data->cnt * 68 / 25) % (sizeof(rb_string) - 1);
	5667	+ size = (cnt * 68 / 25) % (sizeof(rb_string) - 1);
4987	5668
4988	5669	len = size + sizeof(struct rb_item);
4989	5670
..	..	@@ -5077,10 +5758,15 @@
5077	5758	static __init int test_ringbuffer(void)
5078	5759	{
5079	5760	struct task_struct *rb_hammer;
5080		- struct ring_buffer *buffer;
	5761	+ struct trace_buffer *buffer;
5081	5762	int cpu;
5082	5763	int ret = 0;
5083	5764
	5765	+ if (security_locked_down(LOCKDOWN_TRACEFS)) {
	5766	+ pr_warn("Lockdown is enabled, skipping ring buffer tests\n");
	5767	+ return 0;
	5768	+ }
	5769	+
5084	5770	pr_info("Running ring buffer tests...\n");
5085	5771
5086	5772	buffer = ring_buffer_alloc(RB_TEST_BUFFER_SIZE, RB_FL_OVERWRITE);