~hc/RK356X_SDK_RELEASE.git

..	..	@@ -11,6 +11,7 @@
11	11	#include <linux/trace_seq.h>
12	12	#include <linux/spinlock.h>
13	13	#include <linux/irq_work.h>
	14	+#include <linux/security.h>
14	15	#include <linux/uaccess.h>
15	16	#include <linux/hardirq.h>
16	17	#include <linux/kthread.h> /* for self test */
..	..	@@ -201,7 +202,7 @@
201	202	case RINGBUF_TYPE_DATA:
202	203	return rb_event_data_length(event);
203	204	default:
204		- BUG();
	205	+ WARN_ON_ONCE(1);
205	206	}
206	207	/* not hit */
207	208	return 0;
..	..	@@ -257,7 +258,7 @@
257	258	{
258	259	if (extended_time(event))
259	260	event = skip_time_extend(event);
260		- BUG_ON(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
	261	+ WARN_ON_ONCE(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
261	262	/* If length is in len field, then array[0] has the data */
262	263	if (event->type_len)
263	264	return (void *)&event->array[0];
..	..	@@ -277,6 +278,9 @@
277	278
278	279	#define for_each_buffer_cpu(buffer, cpu) \
279	280	for_each_cpu(cpu, buffer->cpumask)
	281	+
	282	+#define for_each_online_buffer_cpu(buffer, cpu) \
	283	+ for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)
280	284
281	285	#define TS_SHIFT 27
282	286	#define TS_MASK ((1ULL << TS_SHIFT) - 1)
..	..	@@ -307,8 +311,6 @@
307	311	#define RB_MISSED_EVENTS (1 << 31)
308	312	/* Missed count stored at end */
309	313	#define RB_MISSED_STORED (1 << 30)
310		-
311		-#define RB_MISSED_FLAGS (RB_MISSED_EVENTS\|RB_MISSED_STORED)
312	314
313	315	struct buffer_data_page {
314	316	u64 time_stamp; /* page time stamp */
..	..	@@ -353,24 +355,11 @@
353	355	local_set(&bpage->commit, 0);
354	356	}
355	357
356		-/**
357		- * ring_buffer_page_len - the size of data on the page.
358		- * @page: The page to read
359		- *
360		- * Returns the amount of data on the page, including buffer page header.
361		- */
362		-size_t ring_buffer_page_len(void *page)
	358	+static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage)
363	359	{
364		- struct buffer_data_page *bpage = page;
365		-
366		- return (local_read(&bpage->commit) & ~RB_MISSED_FLAGS)
367		- + BUF_PAGE_HDR_SIZE;
	360	+ return local_read(&bpage->page->commit);
368	361	}
369	362
370		-/*
371		- * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
372		- * this issue out.
373		- */
374	363	static void free_buffer_page(struct buffer_page *bpage)
375	364	{
376	365	free_page((unsigned long)bpage->page);
..	..	@@ -426,6 +415,7 @@
426	415	struct irq_work work;
427	416	wait_queue_head_t waiters;
428	417	wait_queue_head_t full_waiters;
	418	+ long wait_index;
429	419	bool waiters_pending;
430	420	bool full_waiters_pending;
431	421	bool wakeup_full;
..	..	@@ -437,11 +427,26 @@
437	427	struct rb_event_info {
438	428	u64 ts;
439	429	u64 delta;
	430	+ u64 before;
	431	+ u64 after;
440	432	unsigned long length;
441	433	struct buffer_page *tail_page;
442	434	int add_timestamp;
443	435	};
444	436
	437	+/*
	438	+ * Used for the add_timestamp
	439	+ * NONE
	440	+ * EXTEND - wants a time extend
	441	+ * ABSOLUTE - the buffer requests all events to have absolute time stamps
	442	+ * FORCE - force a full time stamp.
	443	+ */
	444	+enum {
	445	+ RB_ADD_STAMP_NONE = 0,
	446	+ RB_ADD_STAMP_EXTEND = BIT(1),
	447	+ RB_ADD_STAMP_ABSOLUTE = BIT(2),
	448	+ RB_ADD_STAMP_FORCE = BIT(3)
	449	+};
445	450	/*
446	451	* Used for which event context the event is in.
447	452	* TRANSITION = 0
..	..	@@ -461,13 +466,36 @@
461	466	RB_CTX_MAX
462	467	};
463	468
	469	+#if BITS_PER_LONG == 32
	470	+#define RB_TIME_32
	471	+#endif
	472	+
	473	+/* To test on 64 bit machines */
	474	+//#define RB_TIME_32
	475	+
	476	+#ifdef RB_TIME_32
	477	+
	478	+struct rb_time_struct {
	479	+ local_t cnt;
	480	+ local_t top;
	481	+ local_t bottom;
	482	+};
	483	+#else
	484	+#include <asm/local64.h>
	485	+struct rb_time_struct {
	486	+ local64_t time;
	487	+};
	488	+#endif
	489	+typedef struct rb_time_struct rb_time_t;
	490	+
464	491	/*
465	492	* head_page == tail_page && head == tail then buffer is empty.
466	493	*/
467	494	struct ring_buffer_per_cpu {
468	495	int cpu;
469	496	atomic_t record_disabled;
470		- struct ring_buffer *buffer;
	497	+ atomic_t resize_disabled;
	498	+ struct trace_buffer *buffer;
471	499	raw_spinlock_t reader_lock; /* serialize readers */
472	500	arch_spinlock_t lock;
473	501	struct lock_class_key lock_key;
..	..	@@ -489,10 +517,18 @@
489	517	local_t dropped_events;
490	518	local_t committing;
491	519	local_t commits;
	520	+ local_t pages_touched;
	521	+ local_t pages_lost;
	522	+ local_t pages_read;
	523	+ long last_pages_touch;
	524	+ size_t shortest_full;
492	525	unsigned long read;
493	526	unsigned long read_bytes;
494		- u64 write_stamp;
	527	+ rb_time_t write_stamp;
	528	+ rb_time_t before_stamp;
495	529	u64 read_stamp;
	530	+ /* pages removed since last reset */
	531	+ unsigned long pages_removed;
496	532	/* ring buffer pages to update, > 0 to add, < 0 to remove */
497	533	long nr_pages_to_update;
498	534	struct list_head new_pages; /* new pages to add */
..	..	@@ -502,11 +538,11 @@
502	538	struct rb_irq_work irq_work;
503	539	};
504	540
505		-struct ring_buffer {
	541	+struct trace_buffer {
506	542	unsigned flags;
507	543	int cpus;
508	544	atomic_t record_disabled;
509		- atomic_t resize_disabled;
	545	+ atomic_t resizing;
510	546	cpumask_var_t cpumask;
511	547
512	548	struct lock_class_key *reader_lock_key;
..	..	@@ -525,11 +561,257 @@
525	561	struct ring_buffer_iter {
526	562	struct ring_buffer_per_cpu *cpu_buffer;
527	563	unsigned long head;
	564	+ unsigned long next_event;
528	565	struct buffer_page *head_page;
529	566	struct buffer_page *cache_reader_page;
530	567	unsigned long cache_read;
	568	+ unsigned long cache_pages_removed;
531	569	u64 read_stamp;
	570	+ u64 page_stamp;
	571	+ struct ring_buffer_event *event;
	572	+ int missed_events;
532	573	};
	574	+
	575	+#ifdef RB_TIME_32
	576	+
	577	+/*
	578	+ * On 32 bit machines, local64_t is very expensive. As the ring
	579	+ * buffer doesn't need all the features of a true 64 bit atomic,
	580	+ * on 32 bit, it uses these functions (64 still uses local64_t).
	581	+ *
	582	+ * For the ring buffer, 64 bit required operations for the time is
	583	+ * the following:
	584	+ *
	585	+ * - Only need 59 bits (uses 60 to make it even).
	586	+ * - Reads may fail if it interrupted a modification of the time stamp.
	587	+ * It will succeed if it did not interrupt another write even if
	588	+ * the read itself is interrupted by a write.
	589	+ * It returns whether it was successful or not.
	590	+ *
	591	+ * - Writes always succeed and will overwrite other writes and writes
	592	+ * that were done by events interrupting the current write.
	593	+ *
	594	+ * - A write followed by a read of the same time stamp will always succeed,
	595	+ * but may not contain the same value.
	596	+ *
	597	+ * - A cmpxchg will fail if it interrupted another write or cmpxchg.
	598	+ * Other than that, it acts like a normal cmpxchg.
	599	+ *
	600	+ * The 60 bit time stamp is broken up by 30 bits in a top and bottom half
	601	+ * (bottom being the least significant 30 bits of the 60 bit time stamp).
	602	+ *
	603	+ * The two most significant bits of each half holds a 2 bit counter (0-3).
	604	+ * Each update will increment this counter by one.
	605	+ * When reading the top and bottom, if the two counter bits match then the
	606	+ * top and bottom together make a valid 60 bit number.
	607	+ */
	608	+#define RB_TIME_SHIFT 30
	609	+#define RB_TIME_VAL_MASK ((1 << RB_TIME_SHIFT) - 1)
	610	+
	611	+static inline int rb_time_cnt(unsigned long val)
	612	+{
	613	+ return (val >> RB_TIME_SHIFT) & 3;
	614	+}
	615	+
	616	+static inline u64 rb_time_val(unsigned long top, unsigned long bottom)
	617	+{
	618	+ u64 val;
	619	+
	620	+ val = top & RB_TIME_VAL_MASK;
	621	+ val <<= RB_TIME_SHIFT;
	622	+ val \|= bottom & RB_TIME_VAL_MASK;
	623	+
	624	+ return val;
	625	+}
	626	+
	627	+static inline bool __rb_time_read(rb_time_t t, u64 ret, unsigned long *cnt)
	628	+{
	629	+ unsigned long top, bottom;
	630	+ unsigned long c;
	631	+
	632	+ /*
	633	+ * If the read is interrupted by a write, then the cnt will
	634	+ * be different. Loop until both top and bottom have been read
	635	+ * without interruption.
	636	+ */
	637	+ do {
	638	+ c = local_read(&t->cnt);
	639	+ top = local_read(&t->top);
	640	+ bottom = local_read(&t->bottom);
	641	+ } while (c != local_read(&t->cnt));
	642	+
	643	+ *cnt = rb_time_cnt(top);
	644	+
	645	+ /* If top and bottom counts don't match, this interrupted a write */
	646	+ if (*cnt != rb_time_cnt(bottom))
	647	+ return false;
	648	+
	649	+ *ret = rb_time_val(top, bottom);
	650	+ return true;
	651	+}
	652	+
	653	+static bool rb_time_read(rb_time_t t, u64 ret)
	654	+{
	655	+ unsigned long cnt;
	656	+
	657	+ return __rb_time_read(t, ret, &cnt);
	658	+}
	659	+
	660	+static inline unsigned long rb_time_val_cnt(unsigned long val, unsigned long cnt)
	661	+{
	662	+ return (val & RB_TIME_VAL_MASK) \| ((cnt & 3) << RB_TIME_SHIFT);
	663	+}
	664	+
	665	+static inline void rb_time_split(u64 val, unsigned long top, unsigned long bottom)
	666	+{
	667	+ *top = (unsigned long)((val >> RB_TIME_SHIFT) & RB_TIME_VAL_MASK);
	668	+ *bottom = (unsigned long)(val & RB_TIME_VAL_MASK);
	669	+}
	670	+
	671	+static inline void rb_time_val_set(local_t *t, unsigned long val, unsigned long cnt)
	672	+{
	673	+ val = rb_time_val_cnt(val, cnt);
	674	+ local_set(t, val);
	675	+}
	676	+
	677	+static void rb_time_set(rb_time_t *t, u64 val)
	678	+{
	679	+ unsigned long cnt, top, bottom;
	680	+
	681	+ rb_time_split(val, &top, &bottom);
	682	+
	683	+ /* Writes always succeed with a valid number even if it gets interrupted. */
	684	+ do {
	685	+ cnt = local_inc_return(&t->cnt);
	686	+ rb_time_val_set(&t->top, top, cnt);
	687	+ rb_time_val_set(&t->bottom, bottom, cnt);
	688	+ } while (cnt != local_read(&t->cnt));
	689	+}
	690	+
	691	+static inline bool
	692	+rb_time_read_cmpxchg(local_t *l, unsigned long expect, unsigned long set)
	693	+{
	694	+ unsigned long ret;
	695	+
	696	+ ret = local_cmpxchg(l, expect, set);
	697	+ return ret == expect;
	698	+}
	699	+
	700	+static int rb_time_cmpxchg(rb_time_t *t, u64 expect, u64 set)
	701	+{
	702	+ unsigned long cnt, top, bottom;
	703	+ unsigned long cnt2, top2, bottom2;
	704	+ u64 val;
	705	+
	706	+ /* The cmpxchg always fails if it interrupted an update */
	707	+ if (!__rb_time_read(t, &val, &cnt2))
	708	+ return false;
	709	+
	710	+ if (val != expect)
	711	+ return false;
	712	+
	713	+ cnt = local_read(&t->cnt);
	714	+ if ((cnt & 3) != cnt2)
	715	+ return false;
	716	+
	717	+ cnt2 = cnt + 1;
	718	+
	719	+ rb_time_split(val, &top, &bottom);
	720	+ top = rb_time_val_cnt(top, cnt);
	721	+ bottom = rb_time_val_cnt(bottom, cnt);
	722	+
	723	+ rb_time_split(set, &top2, &bottom2);
	724	+ top2 = rb_time_val_cnt(top2, cnt2);
	725	+ bottom2 = rb_time_val_cnt(bottom2, cnt2);
	726	+
	727	+ if (!rb_time_read_cmpxchg(&t->cnt, cnt, cnt2))
	728	+ return false;
	729	+ if (!rb_time_read_cmpxchg(&t->top, top, top2))
	730	+ return false;
	731	+ if (!rb_time_read_cmpxchg(&t->bottom, bottom, bottom2))
	732	+ return false;
	733	+ return true;
	734	+}
	735	+
	736	+#else /* 64 bits */
	737	+
	738	+/* local64_t always succeeds */
	739	+
	740	+static inline bool rb_time_read(rb_time_t t, u64 ret)
	741	+{
	742	+ *ret = local64_read(&t->time);
	743	+ return true;
	744	+}
	745	+static void rb_time_set(rb_time_t *t, u64 val)
	746	+{
	747	+ local64_set(&t->time, val);
	748	+}
	749	+
	750	+static bool rb_time_cmpxchg(rb_time_t *t, u64 expect, u64 set)
	751	+{
	752	+ u64 val;
	753	+ val = local64_cmpxchg(&t->time, expect, set);
	754	+ return val == expect;
	755	+}
	756	+#endif
	757	+
	758	+/**
	759	+ * ring_buffer_nr_pages - get the number of buffer pages in the ring buffer
	760	+ * @buffer: The ring_buffer to get the number of pages from
	761	+ * @cpu: The cpu of the ring_buffer to get the number of pages from
	762	+ *
	763	+ * Returns the number of pages used by a per_cpu buffer of the ring buffer.
	764	+ */
	765	+size_t ring_buffer_nr_pages(struct trace_buffer *buffer, int cpu)
	766	+{
	767	+ return buffer->buffers[cpu]->nr_pages;
	768	+}
	769	+
	770	+/**
	771	+ * ring_buffer_nr_pages_dirty - get the number of used pages in the ring buffer
	772	+ * @buffer: The ring_buffer to get the number of pages from
	773	+ * @cpu: The cpu of the ring_buffer to get the number of pages from
	774	+ *
	775	+ * Returns the number of pages that have content in the ring buffer.
	776	+ */
	777	+size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu)
	778	+{
	779	+ size_t read;
	780	+ size_t lost;
	781	+ size_t cnt;
	782	+
	783	+ read = local_read(&buffer->buffers[cpu]->pages_read);
	784	+ lost = local_read(&buffer->buffers[cpu]->pages_lost);
	785	+ cnt = local_read(&buffer->buffers[cpu]->pages_touched);
	786	+
	787	+ if (WARN_ON_ONCE(cnt < lost))
	788	+ return 0;
	789	+
	790	+ cnt -= lost;
	791	+
	792	+ /* The reader can read an empty page, but not more than that */
	793	+ if (cnt < read) {
	794	+ WARN_ON_ONCE(read > cnt + 1);
	795	+ return 0;
	796	+ }
	797	+
	798	+ return cnt - read;
	799	+}
	800	+
	801	+static __always_inline bool full_hit(struct trace_buffer *buffer, int cpu, int full)
	802	+{
	803	+ struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	804	+ size_t nr_pages;
	805	+ size_t dirty;
	806	+
	807	+ nr_pages = cpu_buffer->nr_pages;
	808	+ if (!nr_pages \|\| !full)
	809	+ return true;
	810	+
	811	+ dirty = ring_buffer_nr_dirty_pages(buffer, cpu);
	812	+
	813	+ return (dirty * 100) > (full * nr_pages);
	814	+}
533	815
534	816	/*
535	817	* rb_wake_up_waiters - wake up tasks waiting for ring buffer input
..	..	@@ -542,27 +824,60 @@
542	824	struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);
543	825
544	826	wake_up_all(&rbwork->waiters);
545		- if (rbwork->wakeup_full) {
	827	+ if (rbwork->full_waiters_pending \|\| rbwork->wakeup_full) {
546	828	rbwork->wakeup_full = false;
	829	+ rbwork->full_waiters_pending = false;
547	830	wake_up_all(&rbwork->full_waiters);
548	831	}
	832	+}
	833	+
	834	+/**
	835	+ * ring_buffer_wake_waiters - wake up any waiters on this ring buffer
	836	+ * @buffer: The ring buffer to wake waiters on
	837	+ *
	838	+ * In the case of a file that represents a ring buffer is closing,
	839	+ * it is prudent to wake up any waiters that are on this.
	840	+ */
	841	+void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu)
	842	+{
	843	+ struct ring_buffer_per_cpu *cpu_buffer;
	844	+ struct rb_irq_work *rbwork;
	845	+
	846	+ if (cpu == RING_BUFFER_ALL_CPUS) {
	847	+
	848	+ /* Wake up individual ones too. One level recursion */
	849	+ for_each_buffer_cpu(buffer, cpu)
	850	+ ring_buffer_wake_waiters(buffer, cpu);
	851	+
	852	+ rbwork = &buffer->irq_work;
	853	+ } else {
	854	+ cpu_buffer = buffer->buffers[cpu];
	855	+ rbwork = &cpu_buffer->irq_work;
	856	+ }
	857	+
	858	+ rbwork->wait_index++;
	859	+ /* make sure the waiters see the new index */
	860	+ smp_wmb();
	861	+
	862	+ rb_wake_up_waiters(&rbwork->work);
549	863	}
550	864
551	865	/**
552	866	* ring_buffer_wait - wait for input to the ring buffer
553	867	* @buffer: buffer to wait on
554	868	* @cpu: the cpu buffer to wait on
555		- * @full: wait until a full page is available, if @cpu != RING_BUFFER_ALL_CPUS
	869	+ * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS
556	870	*
557	871	* If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
558	872	* as data is added to any of the @buffer's cpu buffers. Otherwise
559	873	* it will wait for data to be added to a specific cpu buffer.
560	874	*/
561		-int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
	875	+int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full)
562	876	{
563		- struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer);
	877	+ struct ring_buffer_per_cpu *cpu_buffer;
564	878	DEFINE_WAIT(wait);
565	879	struct rb_irq_work *work;
	880	+ long wait_index;
566	881	int ret = 0;
567	882
568	883	/*
..	..	@@ -573,7 +888,7 @@
573	888	if (cpu == RING_BUFFER_ALL_CPUS) {
574	889	work = &buffer->irq_work;
575	890	/* Full only makes sense on per cpu reads */
576		- full = false;
	891	+ full = 0;
577	892	} else {
578	893	if (!cpumask_test_cpu(cpu, buffer->cpumask))
579	894	return -ENODEV;
..	..	@@ -581,6 +896,7 @@
581	896	work = &cpu_buffer->irq_work;
582	897	}
583	898
	899	+ wait_index = READ_ONCE(work->wait_index);
584	900
585	901	while (true) {
586	902	if (full)
..	..	@@ -625,19 +941,29 @@
625	941	!ring_buffer_empty_cpu(buffer, cpu)) {
626	942	unsigned long flags;
627	943	bool pagebusy;
	944	+ bool done;
628	945
629	946	if (!full)
630	947	break;
631	948
632	949	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
633	950	pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
634		- raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
	951	+ done = !pagebusy && full_hit(buffer, cpu, full);
635	952
636		- if (!pagebusy)
	953	+ if (!cpu_buffer->shortest_full \|\|
	954	+ cpu_buffer->shortest_full > full)
	955	+ cpu_buffer->shortest_full = full;
	956	+ raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
	957	+ if (done)
637	958	break;
638	959	}
639	960
640	961	schedule();
	962	+
	963	+ /* Make sure to see the new wait index */
	964	+ smp_rmb();
	965	+ if (wait_index != work->wait_index)
	966	+ break;
641	967	}
642	968
643	969	if (full)
..	..	@@ -654,6 +980,7 @@
654	980	* @cpu: the cpu buffer to wait on
655	981	* @filp: the file descriptor
656	982	* @poll_table: The poll descriptor
	983	+ * @full: wait until the percentage of pages are available, if @cpu != RING_BUFFER_ALL_CPUS
657	984	*
658	985	* If @cpu == RING_BUFFER_ALL_CPUS then the task will wake up as soon
659	986	* as data is added to any of the @buffer's cpu buffers. Otherwise
..	..	@@ -662,15 +989,16 @@
662	989	* Returns EPOLLIN \| EPOLLRDNORM if data exists in the buffers,
663	990	* zero otherwise.
664	991	*/
665		-__poll_t ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu,
666		- struct file filp, poll_table poll_table)
	992	+__poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu,
	993	+ struct file filp, poll_table poll_table, int full)
667	994	{
668	995	struct ring_buffer_per_cpu *cpu_buffer;
669	996	struct rb_irq_work *work;
670	997
671		- if (cpu == RING_BUFFER_ALL_CPUS)
	998	+ if (cpu == RING_BUFFER_ALL_CPUS) {
672	999	work = &buffer->irq_work;
673		- else {
	1000	+ full = 0;
	1001	+ } else {
674	1002	if (!cpumask_test_cpu(cpu, buffer->cpumask))
675	1003	return -EINVAL;
676	1004
..	..	@@ -678,8 +1006,17 @@
678	1006	work = &cpu_buffer->irq_work;
679	1007	}
680	1008
681		- poll_wait(filp, &work->waiters, poll_table);
682		- work->waiters_pending = true;
	1009	+ if (full) {
	1010	+ poll_wait(filp, &work->full_waiters, poll_table);
	1011	+ work->full_waiters_pending = true;
	1012	+ if (!cpu_buffer->shortest_full \|\|
	1013	+ cpu_buffer->shortest_full > full)
	1014	+ cpu_buffer->shortest_full = full;
	1015	+ } else {
	1016	+ poll_wait(filp, &work->waiters, poll_table);
	1017	+ work->waiters_pending = true;
	1018	+ }
	1019	+
683	1020	/*
684	1021	* There's a tight race between setting the waiters_pending and
685	1022	* checking if the ring buffer is empty. Once the waiters_pending bit
..	..	@@ -694,6 +1031,9 @@
694	1031	* will fix it later.
695	1032	*/
696	1033	smp_mb();
	1034	+
	1035	+ if (full)
	1036	+ return full_hit(buffer, cpu, full) ? EPOLLIN \| EPOLLRDNORM : 0;
697	1037
698	1038	if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) \|\|
699	1039	(cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))
..	..	@@ -720,13 +1060,21 @@
720	1060	/* Up this if you want to test the TIME_EXTENTS and normalization */
721	1061	#define DEBUG_SHIFT 0
722	1062
723		-static inline u64 rb_time_stamp(struct ring_buffer *buffer)
	1063	+static inline u64 rb_time_stamp(struct trace_buffer *buffer)
724	1064	{
	1065	+ u64 ts;
	1066	+
	1067	+ /* Skip retpolines :-( */
	1068	+ if (IS_ENABLED(CONFIG_RETPOLINE) && likely(buffer->clock == trace_clock_local))
	1069	+ ts = trace_clock_local();
	1070	+ else
	1071	+ ts = buffer->clock();
	1072	+
725	1073	/* shift to debug/test normalization and TIME_EXTENTS */
726		- return buffer->clock() << DEBUG_SHIFT;
	1074	+ return ts << DEBUG_SHIFT;
727	1075	}
728	1076
729		-u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu)
	1077	+u64 ring_buffer_time_stamp(struct trace_buffer *buffer, int cpu)
730	1078	{
731	1079	u64 time;
732	1080
..	..	@@ -738,7 +1086,7 @@
738	1086	}
739	1087	EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);
740	1088
741		-void ring_buffer_normalize_time_stamp(struct ring_buffer *buffer,
	1089	+void ring_buffer_normalize_time_stamp(struct trace_buffer *buffer,
742	1090	int cpu, u64 *ts)
743	1091	{
744	1092	/* Just stupid testing the normalize function and deltas */
..	..	@@ -1056,6 +1404,7 @@
1056	1404	old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
1057	1405	old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);
1058	1406
	1407	+ local_inc(&cpu_buffer->pages_touched);
1059	1408	/*
1060	1409	* Just make sure we have seen our old_write and synchronize
1061	1410	* with any interrupts that come in.
..	..	@@ -1109,19 +1458,6 @@
1109	1458	}
1110	1459
1111	1460	/**
1112		- * rb_check_list - make sure a pointer to a list has the last bits zero
1113		- */
1114		-static int rb_check_list(struct ring_buffer_per_cpu *cpu_buffer,
1115		- struct list_head *list)
1116		-{
1117		- if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev) != list->prev))
1118		- return 1;
1119		- if (RB_WARN_ON(cpu_buffer, rb_list_head(list->next) != list->next))
1120		- return 1;
1121		- return 0;
1122		-}
1123		-
1124		-/**
1125	1461	* rb_check_pages - integrity check of buffer pages
1126	1462	* @cpu_buffer: CPU buffer with pages to test
1127	1463	*
..	..	@@ -1130,35 +1466,26 @@
1130	1466	*/
1131	1467	static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
1132	1468	{
1133		- struct list_head *head = cpu_buffer->pages;
1134		- struct buffer_page bpage, tmp;
	1469	+ struct list_head *head = rb_list_head(cpu_buffer->pages);
	1470	+ struct list_head *tmp;
1135	1471
1136		- /* Reset the head page if it exists */
1137		- if (cpu_buffer->head_page)
1138		- rb_set_head_page(cpu_buffer);
1139		-
1140		- rb_head_page_deactivate(cpu_buffer);
1141		-
1142		- if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
1143		- return -1;
1144		- if (RB_WARN_ON(cpu_buffer, head->prev->next != head))
	1472	+ if (RB_WARN_ON(cpu_buffer,
	1473	+ rb_list_head(rb_list_head(head->next)->prev) != head))
1145	1474	return -1;
1146	1475
1147		- if (rb_check_list(cpu_buffer, head))
	1476	+ if (RB_WARN_ON(cpu_buffer,
	1477	+ rb_list_head(rb_list_head(head->prev)->next) != head))
1148	1478	return -1;
1149	1479
1150		- list_for_each_entry_safe(bpage, tmp, head, list) {
	1480	+ for (tmp = rb_list_head(head->next); tmp != head; tmp = rb_list_head(tmp->next)) {
1151	1481	if (RB_WARN_ON(cpu_buffer,
1152		- bpage->list.next->prev != &bpage->list))
	1482	+ rb_list_head(rb_list_head(tmp->next)->prev) != tmp))
1153	1483	return -1;
	1484	+
1154	1485	if (RB_WARN_ON(cpu_buffer,
1155		- bpage->list.prev->next != &bpage->list))
1156		- return -1;
1157		- if (rb_check_list(cpu_buffer, &bpage->list))
	1486	+ rb_list_head(rb_list_head(tmp->prev)->next) != tmp))
1158	1487	return -1;
1159	1488	}
1160		-
1161		- rb_head_page_activate(cpu_buffer);
1162	1489
1163	1490	return 0;
1164	1491	}
..	..	@@ -1260,7 +1587,7 @@
1260	1587	}
1261	1588
1262	1589	static struct ring_buffer_per_cpu *
1263		-rb_allocate_cpu_buffer(struct ring_buffer *buffer, long nr_pages, int cpu)
	1590	+rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu)
1264	1591	{
1265	1592	struct ring_buffer_per_cpu *cpu_buffer;
1266	1593	struct buffer_page *bpage;
..	..	@@ -1325,11 +1652,13 @@
1325	1652	struct list_head *head = cpu_buffer->pages;
1326	1653	struct buffer_page bpage, tmp;
1327	1654
	1655	+ irq_work_sync(&cpu_buffer->irq_work.work);
	1656	+
1328	1657	free_buffer_page(cpu_buffer->reader_page);
1329	1658
1330		- rb_head_page_deactivate(cpu_buffer);
1331		-
1332	1659	if (head) {
	1660	+ rb_head_page_deactivate(cpu_buffer);
	1661	+
1333	1662	list_for_each_entry_safe(bpage, tmp, head, list) {
1334	1663	list_del_init(&bpage->list);
1335	1664	free_buffer_page(bpage);
..	..	@@ -1345,16 +1674,17 @@
1345	1674	* __ring_buffer_alloc - allocate a new ring_buffer
1346	1675	* @size: the size in bytes per cpu that is needed.
1347	1676	* @flags: attributes to set for the ring buffer.
	1677	+ * @key: ring buffer reader_lock_key.
1348	1678	*
1349	1679	* Currently the only flag that is available is the RB_FL_OVERWRITE
1350	1680	* flag. This flag means that the buffer will overwrite old data
1351	1681	* when the buffer wraps. If this flag is not set, the buffer will
1352	1682	* drop data when the tail hits the head.
1353	1683	*/
1354		-struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
	1684	+struct trace_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
1355	1685	struct lock_class_key *key)
1356	1686	{
1357		- struct ring_buffer *buffer;
	1687	+ struct trace_buffer *buffer;
1358	1688	long nr_pages;
1359	1689	int bsize;
1360	1690	int cpu;
..	..	@@ -1424,11 +1754,13 @@
1424	1754	* @buffer: the buffer to free.
1425	1755	*/
1426	1756	void
1427		-ring_buffer_free(struct ring_buffer *buffer)
	1757	+ring_buffer_free(struct trace_buffer *buffer)
1428	1758	{
1429	1759	int cpu;
1430	1760
1431	1761	cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);
	1762	+
	1763	+ irq_work_sync(&buffer->irq_work.work);
1432	1764
1433	1765	for_each_buffer_cpu(buffer, cpu)
1434	1766	rb_free_cpu_buffer(buffer->buffers[cpu]);
..	..	@@ -1440,18 +1772,18 @@
1440	1772	}
1441	1773	EXPORT_SYMBOL_GPL(ring_buffer_free);
1442	1774
1443		-void ring_buffer_set_clock(struct ring_buffer *buffer,
	1775	+void ring_buffer_set_clock(struct trace_buffer *buffer,
1444	1776	u64 (*clock)(void))
1445	1777	{
1446	1778	buffer->clock = clock;
1447	1779	}
1448	1780
1449		-void ring_buffer_set_time_stamp_abs(struct ring_buffer *buffer, bool abs)
	1781	+void ring_buffer_set_time_stamp_abs(struct trace_buffer *buffer, bool abs)
1450	1782	{
1451	1783	buffer->time_stamp_abs = abs;
1452	1784	}
1453	1785
1454		-bool ring_buffer_time_stamp_abs(struct ring_buffer *buffer)
	1786	+bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer)
1455	1787	{
1456	1788	return buffer->time_stamp_abs;
1457	1789	}
..	..	@@ -1509,6 +1841,8 @@
1509	1841	to_remove = rb_list_head(to_remove)->next;
1510	1842	head_bit \|= (unsigned long)to_remove & RB_PAGE_HEAD;
1511	1843	}
	1844	+ /* Read iterators need to reset themselves when some pages removed */
	1845	+ cpu_buffer->pages_removed += nr_removed;
1512	1846
1513	1847	next_page = rb_list_head(to_remove)->next;
1514	1848
..	..	@@ -1529,12 +1863,6 @@
1529	1863	if (head_bit)
1530	1864	cpu_buffer->head_page = list_entry(next_page,
1531	1865	struct buffer_page, list);
1532		-
1533		- /*
1534		- * change read pointer to make sure any read iterators reset
1535		- * themselves
1536		- */
1537		- cpu_buffer->read = 0;
1538	1866
1539	1867	/* pages are removed, resume tracing and then free the pages */
1540	1868	atomic_dec(&cpu_buffer->record_disabled);
..	..	@@ -1563,7 +1891,8 @@
1563	1891	* Increment overrun to account for the lost events.
1564	1892	*/
1565	1893	local_add(page_entries, &cpu_buffer->overrun);
1566		- local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
	1894	+ local_sub(rb_page_commit(to_remove_page), &cpu_buffer->entries_bytes);
	1895	+ local_inc(&cpu_buffer->pages_lost);
1567	1896	}
1568	1897
1569	1898	/*
..	..	@@ -1689,7 +2018,7 @@
1689	2018	*
1690	2019	* Returns 0 on success and < 0 on failure.
1691	2020	*/
1692		-int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size,
	2021	+int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size,
1693	2022	int cpu_id)
1694	2023	{
1695	2024	struct ring_buffer_per_cpu *cpu_buffer;
..	..	@@ -1715,18 +2044,24 @@
1715	2044
1716	2045	size = nr_pages * BUF_PAGE_SIZE;
1717	2046
1718		- /*
1719		- * Don't succeed if resizing is disabled, as a reader might be
1720		- * manipulating the ring buffer and is expecting a sane state while
1721		- * this is true.
1722		- */
1723		- if (atomic_read(&buffer->resize_disabled))
1724		- return -EBUSY;
1725		-
1726	2047	/* prevent another thread from changing buffer sizes */
1727	2048	mutex_lock(&buffer->mutex);
	2049	+ atomic_inc(&buffer->resizing);
1728	2050
1729	2051	if (cpu_id == RING_BUFFER_ALL_CPUS) {
	2052	+ /*
	2053	+ * Don't succeed if resizing is disabled, as a reader might be
	2054	+ * manipulating the ring buffer and is expecting a sane state while
	2055	+ * this is true.
	2056	+ */
	2057	+ for_each_buffer_cpu(buffer, cpu) {
	2058	+ cpu_buffer = buffer->buffers[cpu];
	2059	+ if (atomic_read(&cpu_buffer->resize_disabled)) {
	2060	+ err = -EBUSY;
	2061	+ goto out_err_unlock;
	2062	+ }
	2063	+ }
	2064	+
1730	2065	/* calculate the pages to update */
1731	2066	for_each_buffer_cpu(buffer, cpu) {
1732	2067	cpu_buffer = buffer->buffers[cpu];
..	..	@@ -1749,6 +2084,8 @@
1749	2084	err = -ENOMEM;
1750	2085	goto out_err;
1751	2086	}
	2087	+
	2088	+ cond_resched();
1752	2089	}
1753	2090
1754	2091	get_online_cpus();
..	..	@@ -1794,6 +2131,16 @@
1794	2131	if (nr_pages == cpu_buffer->nr_pages)
1795	2132	goto out;
1796	2133
	2134	+ /*
	2135	+ * Don't succeed if resizing is disabled, as a reader might be
	2136	+ * manipulating the ring buffer and is expecting a sane state while
	2137	+ * this is true.
	2138	+ */
	2139	+ if (atomic_read(&cpu_buffer->resize_disabled)) {
	2140	+ err = -EBUSY;
	2141	+ goto out_err_unlock;
	2142	+ }
	2143	+
1797	2144	cpu_buffer->nr_pages_to_update = nr_pages -
1798	2145	cpu_buffer->nr_pages;
1799	2146
..	..	@@ -1836,7 +2183,7 @@
1836	2183	* There could have been a race between checking
1837	2184	* record_disable and incrementing it.
1838	2185	*/
1839		- synchronize_sched();
	2186	+ synchronize_rcu();
1840	2187	for_each_buffer_cpu(buffer, cpu) {
1841	2188	cpu_buffer = buffer->buffers[cpu];
1842	2189	rb_check_pages(cpu_buffer);
..	..	@@ -1844,6 +2191,7 @@
1844	2191	atomic_dec(&buffer->record_disabled);
1845	2192	}
1846	2193
	2194	+ atomic_dec(&buffer->resizing);
1847	2195	mutex_unlock(&buffer->mutex);
1848	2196	return 0;
1849	2197
..	..	@@ -1863,12 +2211,14 @@
1863	2211	free_buffer_page(bpage);
1864	2212	}
1865	2213	}
	2214	+ out_err_unlock:
	2215	+ atomic_dec(&buffer->resizing);
1866	2216	mutex_unlock(&buffer->mutex);
1867	2217	return err;
1868	2218	}
1869	2219	EXPORT_SYMBOL_GPL(ring_buffer_resize);
1870	2220
1871		-void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val)
	2221	+void ring_buffer_change_overwrite(struct trace_buffer *buffer, int val)
1872	2222	{
1873	2223	mutex_lock(&buffer->mutex);
1874	2224	if (val)
..	..	@@ -1891,15 +2241,63 @@
1891	2241	cpu_buffer->reader_page->read);
1892	2242	}
1893	2243
1894		-static __always_inline struct ring_buffer_event *
	2244	+static struct ring_buffer_event *
1895	2245	rb_iter_head_event(struct ring_buffer_iter *iter)
1896	2246	{
1897		- return __rb_page_index(iter->head_page, iter->head);
1898		-}
	2247	+ struct ring_buffer_event *event;
	2248	+ struct buffer_page *iter_head_page = iter->head_page;
	2249	+ unsigned long commit;
	2250	+ unsigned length;
1899	2251
1900		-static __always_inline unsigned rb_page_commit(struct buffer_page *bpage)
1901		-{
1902		- return local_read(&bpage->page->commit);
	2252	+ if (iter->head != iter->next_event)
	2253	+ return iter->event;
	2254	+
	2255	+ /*
	2256	+ * When the writer goes across pages, it issues a cmpxchg which
	2257	+ * is a mb(), which will synchronize with the rmb here.
	2258	+ * (see rb_tail_page_update() and __rb_reserve_next())
	2259	+ */
	2260	+ commit = rb_page_commit(iter_head_page);
	2261	+ smp_rmb();
	2262	+
	2263	+ /* An event needs to be at least 8 bytes in size */
	2264	+ if (iter->head > commit - 8)
	2265	+ goto reset;
	2266	+
	2267	+ event = __rb_page_index(iter_head_page, iter->head);
	2268	+ length = rb_event_length(event);
	2269	+
	2270	+ /*
	2271	+ * READ_ONCE() doesn't work on functions and we don't want the
	2272	+ * compiler doing any crazy optimizations with length.
	2273	+ */
	2274	+ barrier();
	2275	+
	2276	+ if ((iter->head + length) > commit \|\| length > BUF_MAX_DATA_SIZE)
	2277	+ /* Writer corrupted the read? */
	2278	+ goto reset;
	2279	+
	2280	+ memcpy(iter->event, event, length);
	2281	+ /*
	2282	+ * If the page stamp is still the same after this rmb() then the
	2283	+ * event was safely copied without the writer entering the page.
	2284	+ */
	2285	+ smp_rmb();
	2286	+
	2287	+ /* Make sure the page didn't change since we read this */
	2288	+ if (iter->page_stamp != iter_head_page->page->time_stamp \|\|
	2289	+ commit > rb_page_commit(iter_head_page))
	2290	+ goto reset;
	2291	+
	2292	+ iter->next_event = iter->head + length;
	2293	+ return iter->event;
	2294	+ reset:
	2295	+ /* Reset to the beginning */
	2296	+ iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp;
	2297	+ iter->head = 0;
	2298	+ iter->next_event = 0;
	2299	+ iter->missed_events = 1;
	2300	+ return NULL;
1903	2301	}
1904	2302
1905	2303	/* Size is determined by what has been committed */
..	..	@@ -1937,8 +2335,9 @@
1937	2335	else
1938	2336	rb_inc_page(cpu_buffer, &iter->head_page);
1939	2337
1940		- iter->read_stamp = iter->head_page->page->time_stamp;
	2338	+ iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp;
1941	2339	iter->head = 0;
	2340	+ iter->next_event = 0;
1942	2341	}
1943	2342
1944	2343	/*
..	..	@@ -1987,7 +2386,8 @@
1987	2386	* the counters.
1988	2387	*/
1989	2388	local_add(entries, &cpu_buffer->overrun);
1990		- local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
	2389	+ local_sub(rb_page_commit(next_page), &cpu_buffer->entries_bytes);
	2390	+ local_inc(&cpu_buffer->pages_lost);
1991	2391
1992	2392	/*
1993	2393	* The entries will be zeroed out when we move the
..	..	@@ -2129,9 +2529,6 @@
2129	2529
2130	2530	event = __rb_page_index(tail_page, tail);
2131	2531
2132		- /* account for padding bytes */
2133		- local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes);
2134		-
2135	2532	/*
2136	2533	* Save the original length to the meta data.
2137	2534	* This will be used by the reader to add lost event
..	..	@@ -2145,7 +2542,8 @@
2145	2542	* write counter enough to allow another writer to slip
2146	2543	* in on this page.
2147	2544	* We put in a discarded commit instead, to make sure
2148		- * that this space is not used again.
	2545	+ * that this space is not used again, and this space will
	2546	+ * not be accounted into 'entries_bytes'.
2149	2547	*
2150	2548	* If we are less than the minimum size, we don't need to
2151	2549	* worry about it.
..	..	@@ -2155,6 +2553,9 @@
2155	2553
2156	2554	/* Mark the rest of the page with padding */
2157	2555	rb_event_set_padding(event);
	2556	+
	2557	+ /* Make sure the padding is visible before the write update */
	2558	+ smp_wmb();
2158	2559
2159	2560	/* Set the write back to the previous setting */
2160	2561	local_sub(length, &tail_page->write);
..	..	@@ -2166,6 +2567,12 @@
2166	2567	event->type_len = RINGBUF_TYPE_PADDING;
2167	2568	/* time delta must be non zero */
2168	2569	event->time_delta = 1;
	2570	+
	2571	+ /* account for padding bytes */
	2572	+ local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes);
	2573	+
	2574	+ /* Make sure the padding is visible before the tail_page->write update */
	2575	+ smp_wmb();
2169	2576
2170	2577	/* Set write to end of buffer */
2171	2578	length = (tail + length) - BUF_PAGE_SIZE;
..	..	@@ -2183,7 +2590,7 @@
2183	2590	{
2184	2591	struct buffer_page *tail_page = info->tail_page;
2185	2592	struct buffer_page *commit_page = cpu_buffer->commit_page;
2186		- struct ring_buffer *buffer = cpu_buffer->buffer;
	2593	+ struct trace_buffer *buffer = cpu_buffer->buffer;
2187	2594	struct buffer_page *next_page;
2188	2595	int ret;
2189	2596
..	..	@@ -2280,8 +2687,8 @@
2280	2687	return NULL;
2281	2688	}
2282	2689
2283		-/* Slow path, do not inline */
2284		-static noinline struct ring_buffer_event *
	2690	+/* Slow path */
	2691	+static struct ring_buffer_event *
2285	2692	rb_add_time_stamp(struct ring_buffer_event *event, u64 delta, bool abs)
2286	2693	{
2287	2694	if (abs)
..	..	@@ -2305,13 +2712,73 @@
2305	2712	static inline bool rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
2306	2713	struct ring_buffer_event *event);
2307	2714
	2715	+#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
	2716	+static inline bool sched_clock_stable(void)
	2717	+{
	2718	+ return true;
	2719	+}
	2720	+#endif
	2721	+
	2722	+static void
	2723	+rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
	2724	+ struct rb_event_info *info)
	2725	+{
	2726	+ u64 write_stamp;
	2727	+
	2728	+ WARN_ONCE(1, "Delta way too big! %llu ts=%llu before=%llu after=%llu write stamp=%llu\n%s",
	2729	+ (unsigned long long)info->delta,
	2730	+ (unsigned long long)info->ts,
	2731	+ (unsigned long long)info->before,
	2732	+ (unsigned long long)info->after,
	2733	+ (unsigned long long)(rb_time_read(&cpu_buffer->write_stamp, &write_stamp) ? write_stamp : 0),
	2734	+ sched_clock_stable() ? "" :
	2735	+ "If you just came from a suspend/resume,\n"
	2736	+ "please switch to the trace global clock:\n"
	2737	+ " echo global > /sys/kernel/debug/tracing/trace_clock\n"
	2738	+ "or add trace_clock=global to the kernel command line\n");
	2739	+}
	2740	+
	2741	+static void rb_add_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
	2742	+ struct ring_buffer_event **event,
	2743	+ struct rb_event_info *info,
	2744	+ u64 *delta,
	2745	+ unsigned int *length)
	2746	+{
	2747	+ bool abs = info->add_timestamp &
	2748	+ (RB_ADD_STAMP_FORCE \| RB_ADD_STAMP_ABSOLUTE);
	2749	+
	2750	+ if (unlikely(info->delta > (1ULL << 59))) {
	2751	+ /* did the clock go backwards */
	2752	+ if (info->before == info->after && info->before > info->ts) {
	2753	+ /* not interrupted */
	2754	+ static int once;
	2755	+
	2756	+ /*
	2757	+ * This is possible with a recalibrating of the TSC.
	2758	+ * Do not produce a call stack, but just report it.
	2759	+ */
	2760	+ if (!once) {
	2761	+ once++;
	2762	+ pr_warn("Ring buffer clock went backwards: %llu -> %llu\n",
	2763	+ info->before, info->ts);
	2764	+ }
	2765	+ } else
	2766	+ rb_check_timestamp(cpu_buffer, info);
	2767	+ if (!abs)
	2768	+ info->delta = 0;
	2769	+ }
	2770	+ event = rb_add_time_stamp(event, info->delta, abs);
	2771	+ *length -= RB_LEN_TIME_EXTEND;
	2772	+ *delta = 0;
	2773	+}
	2774	+
2308	2775	/**
2309	2776	* rb_update_event - update event type and data
	2777	+ * @cpu_buffer: The per cpu buffer of the @event
2310	2778	* @event: the event to update
2311		- * @type: the type of event
2312		- * @length: the size of the event field in the ring buffer
	2779	+ * @info: The info to update the @event with (contains length and delta)
2313	2780	*
2314		- * Update the type and data fields of the event. The length
	2781	+ * Update the type and data fields of the @event. The length
2315	2782	* is the actual size that is written to the ring buffer,
2316	2783	* and with this, we can determine what to place into the
2317	2784	* data field.
..	..	@@ -2324,21 +2791,12 @@
2324	2791	unsigned length = info->length;
2325	2792	u64 delta = info->delta;
2326	2793
2327		- /* Only a commit updates the timestamp */
2328		- if (unlikely(!rb_event_is_commit(cpu_buffer, event)))
2329		- delta = 0;
2330		-
2331	2794	/*
2332	2795	* If we need to add a timestamp, then we
2333	2796	* add it to the start of the reserved space.
2334	2797	*/
2335		- if (unlikely(info->add_timestamp)) {
2336		- bool abs = ring_buffer_time_stamp_abs(cpu_buffer->buffer);
2337		-
2338		- event = rb_add_time_stamp(event, abs ? info->delta : delta, abs);
2339		- length -= RB_LEN_TIME_EXTEND;
2340		- delta = 0;
2341		- }
	2798	+ if (unlikely(info->add_timestamp))
	2799	+ rb_add_timestamp(cpu_buffer, &event, info, &delta, &length);
2342	2800
2343	2801	event->time_delta = delta;
2344	2802	length -= RB_EVNT_HDR_SIZE;
..	..	@@ -2381,12 +2839,38 @@
2381	2839	return length;
2382	2840	}
2383	2841
2384		-#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
2385		-static inline bool sched_clock_stable(void)
	2842	+static __always_inline bool
	2843	+rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
	2844	+ struct ring_buffer_event *event)
2386	2845	{
2387		- return true;
	2846	+ unsigned long addr = (unsigned long)event;
	2847	+ unsigned long index;
	2848	+
	2849	+ index = rb_event_index(event);
	2850	+ addr &= PAGE_MASK;
	2851	+
	2852	+ return cpu_buffer->commit_page->page == (void *)addr &&
	2853	+ rb_commit_index(cpu_buffer) == index;
2388	2854	}
2389		-#endif
	2855	+
	2856	+static u64 rb_time_delta(struct ring_buffer_event *event)
	2857	+{
	2858	+ switch (event->type_len) {
	2859	+ case RINGBUF_TYPE_PADDING:
	2860	+ return 0;
	2861	+
	2862	+ case RINGBUF_TYPE_TIME_EXTEND:
	2863	+ return ring_buffer_event_time_stamp(event);
	2864	+
	2865	+ case RINGBUF_TYPE_TIME_STAMP:
	2866	+ return 0;
	2867	+
	2868	+ case RINGBUF_TYPE_DATA:
	2869	+ return event->time_delta;
	2870	+ default:
	2871	+ return 0;
	2872	+ }
	2873	+}
2390	2874
2391	2875	static inline int
2392	2876	rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
..	..	@@ -2396,6 +2880,8 @@
2396	2880	struct buffer_page *bpage;
2397	2881	unsigned long index;
2398	2882	unsigned long addr;
	2883	+ u64 write_stamp;
	2884	+ u64 delta;
2399	2885
2400	2886	new_index = rb_event_index(event);
2401	2887	old_index = new_index + rb_event_ts_length(event);
..	..	@@ -2404,10 +2890,43 @@
2404	2890
2405	2891	bpage = READ_ONCE(cpu_buffer->tail_page);
2406	2892
	2893	+ delta = rb_time_delta(event);
	2894	+
	2895	+ if (!rb_time_read(&cpu_buffer->write_stamp, &write_stamp))
	2896	+ return 0;
	2897	+
	2898	+ /* Make sure the write stamp is read before testing the location */
	2899	+ barrier();
	2900	+
2407	2901	if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
2408	2902	unsigned long write_mask =
2409	2903	local_read(&bpage->write) & ~RB_WRITE_MASK;
2410	2904	unsigned long event_length = rb_event_length(event);
	2905	+
	2906	+ /* Something came in, can't discard */
	2907	+ if (!rb_time_cmpxchg(&cpu_buffer->write_stamp,
	2908	+ write_stamp, write_stamp - delta))
	2909	+ return 0;
	2910	+
	2911	+ /*
	2912	+ * It's possible that the event time delta is zero
	2913	+ * (has the same time stamp as the previous event)
	2914	+ * in which case write_stamp and before_stamp could
	2915	+ * be the same. In such a case, force before_stamp
	2916	+ * to be different than write_stamp. It doesn't
	2917	+ * matter what it is, as long as its different.
	2918	+ */
	2919	+ if (!delta)
	2920	+ rb_time_set(&cpu_buffer->before_stamp, 0);
	2921	+
	2922	+ /*
	2923	+ * If an event were to come in now, it would see that the
	2924	+ * write_stamp and the before_stamp are different, and assume
	2925	+ * that this event just added itself before updating
	2926	+ * the write stamp. The interrupting event will fix the
	2927	+ * write stamp for us, and use the before stamp as its delta.
	2928	+ */
	2929	+
2411	2930	/*
2412	2931	* This is on the tail page. It is possible that
2413	2932	* a write could come in and move the tail page
..	..	@@ -2456,19 +2975,21 @@
2456	2975	if (RB_WARN_ON(cpu_buffer,
2457	2976	rb_is_reader_page(cpu_buffer->tail_page)))
2458	2977	return;
	2978	+ /*
	2979	+ * No need for a memory barrier here, as the update
	2980	+ * of the tail_page did it for this page.
	2981	+ */
2459	2982	local_set(&cpu_buffer->commit_page->page->commit,
2460	2983	rb_page_write(cpu_buffer->commit_page));
2461	2984	rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
2462		- /* Only update the write stamp if the page has an event */
2463		- if (rb_page_write(cpu_buffer->commit_page))
2464		- cpu_buffer->write_stamp =
2465		- cpu_buffer->commit_page->page->time_stamp;
2466	2985	/* add barrier to keep gcc from optimizing too much */
2467	2986	barrier();
2468	2987	}
2469	2988	while (rb_commit_index(cpu_buffer) !=
2470	2989	rb_page_write(cpu_buffer->commit_page)) {
2471	2990
	2991	+ /* Make sure the readers see the content of what is committed. */
	2992	+ smp_wmb();
2472	2993	local_set(&cpu_buffer->commit_page->page->commit,
2473	2994	rb_page_write(cpu_buffer->commit_page));
2474	2995	RB_WARN_ON(cpu_buffer,
..	..	@@ -2534,62 +3055,16 @@
2534	3055	event->time_delta = 1;
2535	3056	}
2536	3057
2537		-static __always_inline bool
2538		-rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
2539		- struct ring_buffer_event *event)
2540		-{
2541		- unsigned long addr = (unsigned long)event;
2542		- unsigned long index;
2543		-
2544		- index = rb_event_index(event);
2545		- addr &= PAGE_MASK;
2546		-
2547		- return cpu_buffer->commit_page->page == (void *)addr &&
2548		- rb_commit_index(cpu_buffer) == index;
2549		-}
2550		-
2551		-static __always_inline void
2552		-rb_update_write_stamp(struct ring_buffer_per_cpu *cpu_buffer,
2553		- struct ring_buffer_event *event)
2554		-{
2555		- u64 delta;
2556		-
2557		- /*
2558		- * The event first in the commit queue updates the
2559		- * time stamp.
2560		- */
2561		- if (rb_event_is_commit(cpu_buffer, event)) {
2562		- /*
2563		- * A commit event that is first on a page
2564		- * updates the write timestamp with the page stamp
2565		- */
2566		- if (!rb_event_index(event))
2567		- cpu_buffer->write_stamp =
2568		- cpu_buffer->commit_page->page->time_stamp;
2569		- else if (event->type_len == RINGBUF_TYPE_TIME_EXTEND) {
2570		- delta = ring_buffer_event_time_stamp(event);
2571		- cpu_buffer->write_stamp += delta;
2572		- } else if (event->type_len == RINGBUF_TYPE_TIME_STAMP) {
2573		- delta = ring_buffer_event_time_stamp(event);
2574		- cpu_buffer->write_stamp = delta;
2575		- } else
2576		- cpu_buffer->write_stamp += event->time_delta;
2577		- }
2578		-}
2579		-
2580	3058	static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
2581	3059	struct ring_buffer_event *event)
2582	3060	{
2583	3061	local_inc(&cpu_buffer->entries);
2584		- rb_update_write_stamp(cpu_buffer, event);
2585	3062	rb_end_commit(cpu_buffer);
2586	3063	}
2587	3064
2588	3065	static __always_inline void
2589		-rb_wakeups(struct ring_buffer buffer, struct ring_buffer_per_cpu cpu_buffer)
	3066	+rb_wakeups(struct trace_buffer buffer, struct ring_buffer_per_cpu cpu_buffer)
2590	3067	{
2591		- bool pagebusy;
2592		-
2593	3068	if (buffer->irq_work.waiters_pending) {
2594	3069	buffer->irq_work.waiters_pending = false;
2595	3070	/* irq_work_queue() supplies it's own memory barriers */
..	..	@@ -2602,14 +3077,24 @@
2602	3077	irq_work_queue(&cpu_buffer->irq_work.work);
2603	3078	}
2604	3079
2605		- pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
	3080	+ if (cpu_buffer->last_pages_touch == local_read(&cpu_buffer->pages_touched))
	3081	+ return;
2606	3082
2607		- if (!pagebusy && cpu_buffer->irq_work.full_waiters_pending) {
2608		- cpu_buffer->irq_work.wakeup_full = true;
2609		- cpu_buffer->irq_work.full_waiters_pending = false;
2610		- /* irq_work_queue() supplies it's own memory barriers */
2611		- irq_work_queue(&cpu_buffer->irq_work.work);
2612		- }
	3083	+ if (cpu_buffer->reader_page == cpu_buffer->commit_page)
	3084	+ return;
	3085	+
	3086	+ if (!cpu_buffer->irq_work.full_waiters_pending)
	3087	+ return;
	3088	+
	3089	+ cpu_buffer->last_pages_touch = local_read(&cpu_buffer->pages_touched);
	3090	+
	3091	+ if (!full_hit(buffer, cpu_buffer->cpu, cpu_buffer->shortest_full))
	3092	+ return;
	3093	+
	3094	+ cpu_buffer->irq_work.wakeup_full = true;
	3095	+ cpu_buffer->irq_work.full_waiters_pending = false;
	3096	+ /* irq_work_queue() supplies it's own memory barriers */
	3097	+ irq_work_queue(&cpu_buffer->irq_work.work);
2613	3098	}
2614	3099
2615	3100	/*
..	..	@@ -2727,7 +3212,7 @@
2727	3212	* Call this function before calling another ring_buffer_lock_reserve() and
2728	3213	* call ring_buffer_nest_end() after the nested ring_buffer_unlock_commit().
2729	3214	*/
2730		-void ring_buffer_nest_start(struct ring_buffer *buffer)
	3215	+void ring_buffer_nest_start(struct trace_buffer *buffer)
2731	3216	{
2732	3217	struct ring_buffer_per_cpu *cpu_buffer;
2733	3218	int cpu;
..	..	@@ -2747,7 +3232,7 @@
2747	3232	* Must be called after ring_buffer_nest_start() and after the
2748	3233	* ring_buffer_unlock_commit().
2749	3234	*/
2750		-void ring_buffer_nest_end(struct ring_buffer *buffer)
	3235	+void ring_buffer_nest_end(struct trace_buffer *buffer)
2751	3236	{
2752	3237	struct ring_buffer_per_cpu *cpu_buffer;
2753	3238	int cpu;
..	..	@@ -2769,7 +3254,7 @@
2769	3254	*
2770	3255	* Must be paired with ring_buffer_lock_reserve.
2771	3256	*/
2772		-int ring_buffer_unlock_commit(struct ring_buffer *buffer,
	3257	+int ring_buffer_unlock_commit(struct trace_buffer *buffer,
2773	3258	struct ring_buffer_event *event)
2774	3259	{
2775	3260	struct ring_buffer_per_cpu *cpu_buffer;
..	..	@@ -2789,57 +3274,135 @@
2789	3274	}
2790	3275	EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);
2791	3276
2792		-static noinline void
2793		-rb_handle_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
2794		- struct rb_event_info *info)
2795		-{
2796		- WARN_ONCE(info->delta > (1ULL << 59),
2797		- KERN_WARNING "Delta way too big! %llu ts=%llu write stamp = %llu\n%s",
2798		- (unsigned long long)info->delta,
2799		- (unsigned long long)info->ts,
2800		- (unsigned long long)cpu_buffer->write_stamp,
2801		- sched_clock_stable() ? "" :
2802		- "If you just came from a suspend/resume,\n"
2803		- "please switch to the trace global clock:\n"
2804		- " echo global > /sys/kernel/debug/tracing/trace_clock\n"
2805		- "or add trace_clock=global to the kernel command line\n");
2806		- info->add_timestamp = 1;
2807		-}
2808		-
2809	3277	static struct ring_buffer_event *
2810	3278	__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
2811	3279	struct rb_event_info *info)
2812	3280	{
2813	3281	struct ring_buffer_event *event;
2814	3282	struct buffer_page *tail_page;
2815		- unsigned long tail, write;
2816		-
2817		- /*
2818		- * If the time delta since the last event is too big to
2819		- * hold in the time field of the event, then we append a
2820		- * TIME EXTEND event ahead of the data event.
2821		- */
2822		- if (unlikely(info->add_timestamp))
2823		- info->length += RB_LEN_TIME_EXTEND;
	3283	+ unsigned long tail, write, w;
	3284	+ bool a_ok;
	3285	+ bool b_ok;
2824	3286
2825	3287	/* Don't let the compiler play games with cpu_buffer->tail_page */
2826	3288	tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page);
2827		- write = local_add_return(info->length, &tail_page->write);
	3289	+
	3290	+ /A/ w = local_read(&tail_page->write) & RB_WRITE_MASK;
	3291	+ barrier();
	3292	+ b_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before);
	3293	+ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);
	3294	+ barrier();
	3295	+ info->ts = rb_time_stamp(cpu_buffer->buffer);
	3296	+
	3297	+ if ((info->add_timestamp & RB_ADD_STAMP_ABSOLUTE)) {
	3298	+ info->delta = info->ts;
	3299	+ } else {
	3300	+ /*
	3301	+ * If interrupting an event time update, we may need an
	3302	+ * absolute timestamp.
	3303	+ * Don't bother if this is the start of a new page (w == 0).
	3304	+ */
	3305	+ if (unlikely(!a_ok \|\| !b_ok \|\| (info->before != info->after && w))) {
	3306	+ info->add_timestamp \|= RB_ADD_STAMP_FORCE \| RB_ADD_STAMP_EXTEND;
	3307	+ info->length += RB_LEN_TIME_EXTEND;
	3308	+ } else {
	3309	+ info->delta = info->ts - info->after;
	3310	+ if (unlikely(test_time_stamp(info->delta))) {
	3311	+ info->add_timestamp \|= RB_ADD_STAMP_EXTEND;
	3312	+ info->length += RB_LEN_TIME_EXTEND;
	3313	+ }
	3314	+ }
	3315	+ }
	3316	+
	3317	+ /B/ rb_time_set(&cpu_buffer->before_stamp, info->ts);
	3318	+
	3319	+ /C/ write = local_add_return(info->length, &tail_page->write);
2828	3320
2829	3321	/* set write to only the index of the write */
2830	3322	write &= RB_WRITE_MASK;
	3323	+
2831	3324	tail = write - info->length;
	3325	+
	3326	+ /* See if we shot pass the end of this buffer page */
	3327	+ if (unlikely(write > BUF_PAGE_SIZE)) {
	3328	+ /* before and after may now different, fix it up*/
	3329	+ b_ok = rb_time_read(&cpu_buffer->before_stamp, &info->before);
	3330	+ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);
	3331	+ if (a_ok && b_ok && info->before != info->after)
	3332	+ (void)rb_time_cmpxchg(&cpu_buffer->before_stamp,
	3333	+ info->before, info->after);
	3334	+ return rb_move_tail(cpu_buffer, tail, info);
	3335	+ }
	3336	+
	3337	+ if (likely(tail == w)) {
	3338	+ u64 save_before;
	3339	+ bool s_ok;
	3340	+
	3341	+ /* Nothing interrupted us between A and C */
	3342	+ /D/ rb_time_set(&cpu_buffer->write_stamp, info->ts);
	3343	+ barrier();
	3344	+ /E/ s_ok = rb_time_read(&cpu_buffer->before_stamp, &save_before);
	3345	+ RB_WARN_ON(cpu_buffer, !s_ok);
	3346	+ if (likely(!(info->add_timestamp &
	3347	+ (RB_ADD_STAMP_FORCE \| RB_ADD_STAMP_ABSOLUTE))))
	3348	+ /* This did not interrupt any time update */
	3349	+ info->delta = info->ts - info->after;
	3350	+ else
	3351	+ /* Just use full timestamp for inerrupting event */
	3352	+ info->delta = info->ts;
	3353	+ barrier();
	3354	+ if (unlikely(info->ts != save_before)) {
	3355	+ /* SLOW PATH - Interrupted between C and E */
	3356	+
	3357	+ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);
	3358	+ RB_WARN_ON(cpu_buffer, !a_ok);
	3359	+
	3360	+ /* Write stamp must only go forward */
	3361	+ if (save_before > info->after) {
	3362	+ /*
	3363	+ * We do not care about the result, only that
	3364	+ * it gets updated atomically.
	3365	+ */
	3366	+ (void)rb_time_cmpxchg(&cpu_buffer->write_stamp,
	3367	+ info->after, save_before);
	3368	+ }
	3369	+ }
	3370	+ } else {
	3371	+ u64 ts;
	3372	+ /* SLOW PATH - Interrupted between A and C */
	3373	+ a_ok = rb_time_read(&cpu_buffer->write_stamp, &info->after);
	3374	+ /* Was interrupted before here, write_stamp must be valid */
	3375	+ RB_WARN_ON(cpu_buffer, !a_ok);
	3376	+ ts = rb_time_stamp(cpu_buffer->buffer);
	3377	+ barrier();
	3378	+ /E/ if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) &&
	3379	+ info->after < ts &&
	3380	+ rb_time_cmpxchg(&cpu_buffer->write_stamp,
	3381	+ info->after, ts)) {
	3382	+ /* Nothing came after this event between C and E */
	3383	+ info->delta = ts - info->after;
	3384	+ info->ts = ts;
	3385	+ } else {
	3386	+ /*
	3387	+ * Interrupted beween C and E:
	3388	+ * Lost the previous events time stamp. Just set the
	3389	+ * delta to zero, and this will be the same time as
	3390	+ * the event this event interrupted. And the events that
	3391	+ * came after this will still be correct (as they would
	3392	+ * have built their delta on the previous event.
	3393	+ */
	3394	+ info->delta = 0;
	3395	+ }
	3396	+ info->add_timestamp &= ~RB_ADD_STAMP_FORCE;
	3397	+ }
2832	3398
2833	3399	/*
2834	3400	* If this is the first commit on the page, then it has the same
2835	3401	* timestamp as the page itself.
2836	3402	*/
2837		- if (!tail && !ring_buffer_time_stamp_abs(cpu_buffer->buffer))
	3403	+ if (unlikely(!tail && !(info->add_timestamp &
	3404	+ (RB_ADD_STAMP_FORCE \| RB_ADD_STAMP_ABSOLUTE))))
2838	3405	info->delta = 0;
2839		-
2840		- /* See if we shot pass the end of this buffer page */
2841		- if (unlikely(write > BUF_PAGE_SIZE))
2842		- return rb_move_tail(cpu_buffer, tail, info);
2843	3406
2844	3407	/* We reserved something on the buffer */
2845	3408
..	..	@@ -2852,7 +3415,7 @@
2852	3415	* If this is the first commit on the page, then update
2853	3416	* its timestamp.
2854	3417	*/
2855		- if (!tail)
	3418	+ if (unlikely(!tail))
2856	3419	tail_page->page->time_stamp = info->ts;
2857	3420
2858	3421	/* account for these added bytes */
..	..	@@ -2862,16 +3425,17 @@
2862	3425	}
2863	3426
2864	3427	static __always_inline struct ring_buffer_event *
2865		-rb_reserve_next_event(struct ring_buffer *buffer,
	3428	+rb_reserve_next_event(struct trace_buffer *buffer,
2866	3429	struct ring_buffer_per_cpu *cpu_buffer,
2867	3430	unsigned long length)
2868	3431	{
2869	3432	struct ring_buffer_event *event;
2870	3433	struct rb_event_info info;
2871	3434	int nr_loops = 0;
2872		- u64 diff;
	3435	+ int add_ts_default;
2873	3436
2874	3437	rb_start_commit(cpu_buffer);
	3438	+ /* The commit page can not change after this */
2875	3439
2876	3440	#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2877	3441	/*
..	..	@@ -2889,8 +3453,16 @@
2889	3453	#endif
2890	3454
2891	3455	info.length = rb_calculate_event_length(length);
	3456	+
	3457	+ if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {
	3458	+ add_ts_default = RB_ADD_STAMP_ABSOLUTE;
	3459	+ info.length += RB_LEN_TIME_EXTEND;
	3460	+ } else {
	3461	+ add_ts_default = RB_ADD_STAMP_NONE;
	3462	+ }
	3463	+
2892	3464	again:
2893		- info.add_timestamp = 0;
	3465	+ info.add_timestamp = add_ts_default;
2894	3466	info.delta = 0;
2895	3467
2896	3468	/*
..	..	@@ -2905,35 +3477,16 @@
2905	3477	if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
2906	3478	goto out_fail;
2907	3479
2908		- info.ts = rb_time_stamp(cpu_buffer->buffer);
2909		- diff = info.ts - cpu_buffer->write_stamp;
2910		-
2911		- /* make sure this diff is calculated here */
2912		- barrier();
2913		-
2914		- if (ring_buffer_time_stamp_abs(buffer)) {
2915		- info.delta = info.ts;
2916		- rb_handle_timestamp(cpu_buffer, &info);
2917		- } else /* Did the write stamp get updated already? */
2918		- if (likely(info.ts >= cpu_buffer->write_stamp)) {
2919		- info.delta = diff;
2920		- if (unlikely(test_time_stamp(info.delta)))
2921		- rb_handle_timestamp(cpu_buffer, &info);
2922		- }
2923		-
2924	3480	event = __rb_reserve_next(cpu_buffer, &info);
2925	3481
2926	3482	if (unlikely(PTR_ERR(event) == -EAGAIN)) {
2927		- if (info.add_timestamp)
	3483	+ if (info.add_timestamp & (RB_ADD_STAMP_FORCE \| RB_ADD_STAMP_EXTEND))
2928	3484	info.length -= RB_LEN_TIME_EXTEND;
2929	3485	goto again;
2930	3486	}
2931	3487
2932		- if (!event)
2933		- goto out_fail;
2934		-
2935		- return event;
2936		-
	3488	+ if (likely(event))
	3489	+ return event;
2937	3490	out_fail:
2938	3491	rb_end_commit(cpu_buffer);
2939	3492	return NULL;
..	..	@@ -2955,7 +3508,7 @@
2955	3508	* If NULL is returned, then nothing has been allocated or locked.
2956	3509	*/
2957	3510	struct ring_buffer_event *
2958		-ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
	3511	+ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length)
2959	3512	{
2960	3513	struct ring_buffer_per_cpu *cpu_buffer;
2961	3514	struct ring_buffer_event *event;
..	..	@@ -3056,7 +3609,7 @@
3056	3609	* If this function is called, do not call ring_buffer_unlock_commit on
3057	3610	* the event.
3058	3611	*/
3059		-void ring_buffer_discard_commit(struct ring_buffer *buffer,
	3612	+void ring_buffer_discard_commit(struct trace_buffer *buffer,
3060	3613	struct ring_buffer_event *event)
3061	3614	{
3062	3615	struct ring_buffer_per_cpu *cpu_buffer;
..	..	@@ -3079,11 +3632,6 @@
3079	3632	if (rb_try_to_discard(cpu_buffer, event))
3080	3633	goto out;
3081	3634
3082		- /*
3083		- * The commit is still visible by the reader, so we
3084		- * must still update the timestamp.
3085		- */
3086		- rb_update_write_stamp(cpu_buffer, event);
3087	3635	out:
3088	3636	rb_end_commit(cpu_buffer);
3089	3637
..	..	@@ -3107,7 +3655,7 @@
3107	3655	* Note, like ring_buffer_lock_reserve, the length is the length of the data
3108	3656	* and not the length of the event which would hold the header.
3109	3657	*/
3110		-int ring_buffer_write(struct ring_buffer *buffer,
	3658	+int ring_buffer_write(struct trace_buffer *buffer,
3111	3659	unsigned long length,
3112	3660	void *data)
3113	3661	{
..	..	@@ -3205,9 +3753,9 @@
3205	3753	* This prevents all writes to the buffer. Any attempt to write
3206	3754	* to the buffer after this will fail and return NULL.
3207	3755	*
3208		- * The caller should call synchronize_sched() after this.
	3756	+ * The caller should call synchronize_rcu() after this.
3209	3757	*/
3210		-void ring_buffer_record_disable(struct ring_buffer *buffer)
	3758	+void ring_buffer_record_disable(struct trace_buffer *buffer)
3211	3759	{
3212	3760	atomic_inc(&buffer->record_disabled);
3213	3761	}
..	..	@@ -3220,7 +3768,7 @@
3220	3768	* Note, multiple disables will need the same number of enables
3221	3769	* to truly enable the writing (much like preempt_disable).
3222	3770	*/
3223		-void ring_buffer_record_enable(struct ring_buffer *buffer)
	3771	+void ring_buffer_record_enable(struct trace_buffer *buffer)
3224	3772	{
3225	3773	atomic_dec(&buffer->record_disabled);
3226	3774	}
..	..	@@ -3237,7 +3785,7 @@
3237	3785	* it works like an on/off switch, where as the disable() version
3238	3786	* must be paired with a enable().
3239	3787	*/
3240		-void ring_buffer_record_off(struct ring_buffer *buffer)
	3788	+void ring_buffer_record_off(struct trace_buffer *buffer)
3241	3789	{
3242	3790	unsigned int rd;
3243	3791	unsigned int new_rd;
..	..	@@ -3260,7 +3808,7 @@
3260	3808	* it works like an on/off switch, where as the enable() version
3261	3809	* must be paired with a disable().
3262	3810	*/
3263		-void ring_buffer_record_on(struct ring_buffer *buffer)
	3811	+void ring_buffer_record_on(struct trace_buffer *buffer)
3264	3812	{
3265	3813	unsigned int rd;
3266	3814	unsigned int new_rd;
..	..	@@ -3278,7 +3826,7 @@
3278	3826	*
3279	3827	* Returns true if the ring buffer is in a state that it accepts writes.
3280	3828	*/
3281		-bool ring_buffer_record_is_on(struct ring_buffer *buffer)
	3829	+bool ring_buffer_record_is_on(struct trace_buffer *buffer)
3282	3830	{
3283	3831	return !atomic_read(&buffer->record_disabled);
3284	3832	}
..	..	@@ -3294,7 +3842,7 @@
3294	3842	* ring_buffer_record_disable(), as that is a temporary disabling of
3295	3843	* the ring buffer.
3296	3844	*/
3297		-bool ring_buffer_record_is_set_on(struct ring_buffer *buffer)
	3845	+bool ring_buffer_record_is_set_on(struct trace_buffer *buffer)
3298	3846	{
3299	3847	return !(atomic_read(&buffer->record_disabled) & RB_BUFFER_OFF);
3300	3848	}
..	..	@@ -3307,9 +3855,9 @@
3307	3855	* This prevents all writes to the buffer. Any attempt to write
3308	3856	* to the buffer after this will fail and return NULL.
3309	3857	*
3310		- * The caller should call synchronize_sched() after this.
	3858	+ * The caller should call synchronize_rcu() after this.
3311	3859	*/
3312		-void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
	3860	+void ring_buffer_record_disable_cpu(struct trace_buffer *buffer, int cpu)
3313	3861	{
3314	3862	struct ring_buffer_per_cpu *cpu_buffer;
3315	3863
..	..	@@ -3329,7 +3877,7 @@
3329	3877	* Note, multiple disables will need the same number of enables
3330	3878	* to truly enable the writing (much like preempt_disable).
3331	3879	*/
3332		-void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
	3880	+void ring_buffer_record_enable_cpu(struct trace_buffer *buffer, int cpu)
3333	3881	{
3334	3882	struct ring_buffer_per_cpu *cpu_buffer;
3335	3883
..	..	@@ -3359,7 +3907,7 @@
3359	3907	* @buffer: The ring buffer
3360	3908	* @cpu: The per CPU buffer to read from.
3361	3909	*/
3362		-u64 ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu)
	3910	+u64 ring_buffer_oldest_event_ts(struct trace_buffer *buffer, int cpu)
3363	3911	{
3364	3912	unsigned long flags;
3365	3913	struct ring_buffer_per_cpu *cpu_buffer;
..	..	@@ -3388,11 +3936,11 @@
3388	3936	EXPORT_SYMBOL_GPL(ring_buffer_oldest_event_ts);
3389	3937
3390	3938	/**
3391		- * ring_buffer_bytes_cpu - get the number of bytes consumed in a cpu buffer
	3939	+ * ring_buffer_bytes_cpu - get the number of bytes unconsumed in a cpu buffer
3392	3940	* @buffer: The ring buffer
3393	3941	* @cpu: The per CPU buffer to read from.
3394	3942	*/
3395		-unsigned long ring_buffer_bytes_cpu(struct ring_buffer *buffer, int cpu)
	3943	+unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu)
3396	3944	{
3397	3945	struct ring_buffer_per_cpu *cpu_buffer;
3398	3946	unsigned long ret;
..	..	@@ -3412,7 +3960,7 @@
3412	3960	* @buffer: The ring buffer
3413	3961	* @cpu: The per CPU buffer to get the entries from.
3414	3962	*/
3415		-unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
	3963	+unsigned long ring_buffer_entries_cpu(struct trace_buffer *buffer, int cpu)
3416	3964	{
3417	3965	struct ring_buffer_per_cpu *cpu_buffer;
3418	3966
..	..	@@ -3431,7 +3979,7 @@
3431	3979	* @buffer: The ring buffer
3432	3980	* @cpu: The per CPU buffer to get the number of overruns from
3433	3981	*/
3434		-unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
	3982	+unsigned long ring_buffer_overrun_cpu(struct trace_buffer *buffer, int cpu)
3435	3983	{
3436	3984	struct ring_buffer_per_cpu *cpu_buffer;
3437	3985	unsigned long ret;
..	..	@@ -3454,7 +4002,7 @@
3454	4002	* @cpu: The per CPU buffer to get the number of overruns from
3455	4003	*/
3456	4004	unsigned long
3457		-ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)
	4005	+ring_buffer_commit_overrun_cpu(struct trace_buffer *buffer, int cpu)
3458	4006	{
3459	4007	struct ring_buffer_per_cpu *cpu_buffer;
3460	4008	unsigned long ret;
..	..	@@ -3476,7 +4024,7 @@
3476	4024	* @cpu: The per CPU buffer to get the number of overruns from
3477	4025	*/
3478	4026	unsigned long
3479		-ring_buffer_dropped_events_cpu(struct ring_buffer *buffer, int cpu)
	4027	+ring_buffer_dropped_events_cpu(struct trace_buffer *buffer, int cpu)
3480	4028	{
3481	4029	struct ring_buffer_per_cpu *cpu_buffer;
3482	4030	unsigned long ret;
..	..	@@ -3497,7 +4045,7 @@
3497	4045	* @cpu: The per CPU buffer to get the number of events read
3498	4046	*/
3499	4047	unsigned long
3500		-ring_buffer_read_events_cpu(struct ring_buffer *buffer, int cpu)
	4048	+ring_buffer_read_events_cpu(struct trace_buffer *buffer, int cpu)
3501	4049	{
3502	4050	struct ring_buffer_per_cpu *cpu_buffer;
3503	4051
..	..	@@ -3516,7 +4064,7 @@
3516	4064	* Returns the total number of entries in the ring buffer
3517	4065	* (all CPU entries)
3518	4066	*/
3519		-unsigned long ring_buffer_entries(struct ring_buffer *buffer)
	4067	+unsigned long ring_buffer_entries(struct trace_buffer *buffer)
3520	4068	{
3521	4069	struct ring_buffer_per_cpu *cpu_buffer;
3522	4070	unsigned long entries = 0;
..	..	@@ -3539,7 +4087,7 @@
3539	4087	* Returns the total number of overruns in the ring buffer
3540	4088	* (all CPU entries)
3541	4089	*/
3542		-unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
	4090	+unsigned long ring_buffer_overruns(struct trace_buffer *buffer)
3543	4091	{
3544	4092	struct ring_buffer_per_cpu *cpu_buffer;
3545	4093	unsigned long overruns = 0;
..	..	@@ -3562,14 +4110,19 @@
3562	4110	/* Iterator usage is expected to have record disabled */
3563	4111	iter->head_page = cpu_buffer->reader_page;
3564	4112	iter->head = cpu_buffer->reader_page->read;
	4113	+ iter->next_event = iter->head;
3565	4114
3566	4115	iter->cache_reader_page = iter->head_page;
3567	4116	iter->cache_read = cpu_buffer->read;
	4117	+ iter->cache_pages_removed = cpu_buffer->pages_removed;
3568	4118
3569		- if (iter->head)
	4119	+ if (iter->head) {
3570	4120	iter->read_stamp = cpu_buffer->read_stamp;
3571		- else
	4121	+ iter->page_stamp = cpu_buffer->reader_page->page->time_stamp;
	4122	+ } else {
3572	4123	iter->read_stamp = iter->head_page->page->time_stamp;
	4124	+ iter->page_stamp = iter->read_stamp;
	4125	+ }
3573	4126	}
3574	4127
3575	4128	/**
..	..	@@ -3605,17 +4158,38 @@
3605	4158	struct buffer_page *reader;
3606	4159	struct buffer_page *head_page;
3607	4160	struct buffer_page *commit_page;
	4161	+ struct buffer_page *curr_commit_page;
3608	4162	unsigned commit;
	4163	+ u64 curr_commit_ts;
	4164	+ u64 commit_ts;
3609	4165
3610	4166	cpu_buffer = iter->cpu_buffer;
3611		-
3612		- /* Remember, trace recording is off when iterator is in use */
3613	4167	reader = cpu_buffer->reader_page;
3614	4168	head_page = cpu_buffer->head_page;
3615	4169	commit_page = cpu_buffer->commit_page;
3616		- commit = rb_page_commit(commit_page);
	4170	+ commit_ts = commit_page->page->time_stamp;
3617	4171
3618		- return ((iter->head_page == commit_page && iter->head == commit) \|\|
	4172	+ /*
	4173	+ * When the writer goes across pages, it issues a cmpxchg which
	4174	+ * is a mb(), which will synchronize with the rmb here.
	4175	+ * (see rb_tail_page_update())
	4176	+ */
	4177	+ smp_rmb();
	4178	+ commit = rb_page_commit(commit_page);
	4179	+ /* We want to make sure that the commit page doesn't change */
	4180	+ smp_rmb();
	4181	+
	4182	+ /* Make sure commit page didn't change */
	4183	+ curr_commit_page = READ_ONCE(cpu_buffer->commit_page);
	4184	+ curr_commit_ts = READ_ONCE(curr_commit_page->page->time_stamp);
	4185	+
	4186	+ /* If the commit page changed, then there's more data */
	4187	+ if (curr_commit_page != commit_page \|\|
	4188	+ curr_commit_ts != commit_ts)
	4189	+ return 0;
	4190	+
	4191	+ /* Still racy, as it may return a false positive, but that's OK */
	4192	+ return ((iter->head_page == commit_page && iter->head >= commit) \|\|
3619	4193	(iter->head_page == reader && commit_page == head_page &&
3620	4194	head_page->read == commit &&
3621	4195	iter->head == rb_page_commit(cpu_buffer->reader_page)));
..	..	@@ -3647,7 +4221,7 @@
3647	4221	return;
3648	4222
3649	4223	default:
3650		- BUG();
	4224	+ RB_WARN_ON(cpu_buffer, 1);
3651	4225	}
3652	4226	return;
3653	4227	}
..	..	@@ -3677,7 +4251,7 @@
3677	4251	return;
3678	4252
3679	4253	default:
3680		- BUG();
	4254	+ RB_WARN_ON(iter->cpu_buffer, 1);
3681	4255	}
3682	4256	return;
3683	4257	}
..	..	@@ -3786,12 +4360,14 @@
3786	4360	goto spin;
3787	4361
3788	4362	/*
3789		- * Yeah! We succeeded in replacing the page.
	4363	+ * Yay! We succeeded in replacing the page.
3790	4364	*
3791	4365	* Now make the new head point back to the reader page.
3792	4366	*/
3793	4367	rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;
3794	4368	rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
	4369	+
	4370	+ local_inc(&cpu_buffer->pages_read);
3795	4371
3796	4372	/* Finally update the reader page to the new head */
3797	4373	cpu_buffer->reader_page = reader;
..	..	@@ -3811,6 +4387,38 @@
3811	4387
3812	4388	arch_spin_unlock(&cpu_buffer->lock);
3813	4389	local_irq_restore(flags);
	4390	+
	4391	+ /*
	4392	+ * The writer has preempt disable, wait for it. But not forever
	4393	+ * Although, 1 second is pretty much "forever"
	4394	+ */
	4395	+#define USECS_WAIT 1000000
	4396	+ for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) {
	4397	+ /* If the write is past the end of page, a writer is still updating it */
	4398	+ if (likely(!reader \|\| rb_page_write(reader) <= BUF_PAGE_SIZE))
	4399	+ break;
	4400	+
	4401	+ udelay(1);
	4402	+
	4403	+ /* Get the latest version of the reader write value */
	4404	+ smp_rmb();
	4405	+ }
	4406	+
	4407	+ /* The writer is not moving forward? Something is wrong */
	4408	+ if (RB_WARN_ON(cpu_buffer, nr_loops == USECS_WAIT))
	4409	+ reader = NULL;
	4410	+
	4411	+ /*
	4412	+ * Make sure we see any padding after the write update
	4413	+ * (see rb_reset_tail()).
	4414	+ *
	4415	+ * In addition, a writer may be writing on the reader page
	4416	+ * if the page has not been fully filled, so the read barrier
	4417	+ * is also needed to make sure we see the content of what is
	4418	+ * committed by the writer (see rb_set_commit_to_write()).
	4419	+ */
	4420	+ smp_rmb();
	4421	+
3814	4422
3815	4423	return reader;
3816	4424	}
..	..	@@ -3836,20 +4444,28 @@
3836	4444
3837	4445	length = rb_event_length(event);
3838	4446	cpu_buffer->reader_page->read += length;
	4447	+ cpu_buffer->read_bytes += length;
3839	4448	}
3840	4449
3841	4450	static void rb_advance_iter(struct ring_buffer_iter *iter)
3842	4451	{
3843	4452	struct ring_buffer_per_cpu *cpu_buffer;
3844		- struct ring_buffer_event *event;
3845		- unsigned length;
3846	4453
3847	4454	cpu_buffer = iter->cpu_buffer;
	4455	+
	4456	+ /* If head == next_event then we need to jump to the next event */
	4457	+ if (iter->head == iter->next_event) {
	4458	+ /* If the event gets overwritten again, there's nothing to do */
	4459	+ if (rb_iter_head_event(iter) == NULL)
	4460	+ return;
	4461	+ }
	4462	+
	4463	+ iter->head = iter->next_event;
3848	4464
3849	4465	/*
3850	4466	* Check if we are at the end of the buffer.
3851	4467	*/
3852		- if (iter->head >= rb_page_size(iter->head_page)) {
	4468	+ if (iter->next_event >= rb_page_size(iter->head_page)) {
3853	4469	/* discarded commits can make the page empty */
3854	4470	if (iter->head_page == cpu_buffer->commit_page)
3855	4471	return;
..	..	@@ -3857,27 +4473,7 @@
3857	4473	return;
3858	4474	}
3859	4475
3860		- event = rb_iter_head_event(iter);
3861		-
3862		- length = rb_event_length(event);
3863		-
3864		- /*
3865		- * This should not be called to advance the header if we are
3866		- * at the tail of the buffer.
3867		- */
3868		- if (RB_WARN_ON(cpu_buffer,
3869		- (iter->head_page == cpu_buffer->commit_page) &&
3870		- (iter->head + length > rb_commit_index(cpu_buffer))))
3871		- return;
3872		-
3873		- rb_update_iter_read_stamp(iter, event);
3874		-
3875		- iter->head += length;
3876		-
3877		- /* check for end of page padding */
3878		- if ((iter->head >= rb_page_size(iter->head_page)) &&
3879		- (iter->head_page != cpu_buffer->commit_page))
3880		- rb_inc_iter(iter);
	4476	+ rb_update_iter_read_stamp(iter, iter->event);
3881	4477	}
3882	4478
3883	4479	static int rb_lost_events(struct ring_buffer_per_cpu *cpu_buffer)
..	..	@@ -3951,7 +4547,7 @@
3951	4547	return event;
3952	4548
3953	4549	default:
3954		- BUG();
	4550	+ RB_WARN_ON(cpu_buffer, 1);
3955	4551	}
3956	4552
3957	4553	return NULL;
..	..	@@ -3961,7 +4557,7 @@
3961	4557	static struct ring_buffer_event *
3962	4558	rb_iter_peek(struct ring_buffer_iter iter, u64 ts)
3963	4559	{
3964		- struct ring_buffer *buffer;
	4560	+ struct trace_buffer *buffer;
3965	4561	struct ring_buffer_per_cpu *cpu_buffer;
3966	4562	struct ring_buffer_event *event;
3967	4563	int nr_loops = 0;
..	..	@@ -3973,12 +4569,13 @@
3973	4569	buffer = cpu_buffer->buffer;
3974	4570
3975	4571	/*
3976		- * Check if someone performed a consuming read to
3977		- * the buffer. A consuming read invalidates the iterator
3978		- * and we need to reset the iterator in this case.
	4572	+ * Check if someone performed a consuming read to the buffer
	4573	+ * or removed some pages from the buffer. In these cases,
	4574	+ * iterator was invalidated and we need to reset it.
3979	4575	*/
3980	4576	if (unlikely(iter->cache_read != cpu_buffer->read \|\|
3981		- iter->cache_reader_page != cpu_buffer->reader_page))
	4577	+ iter->cache_reader_page != cpu_buffer->reader_page \|\|
	4578	+ iter->cache_pages_removed != cpu_buffer->pages_removed))
3982	4579	rb_iter_reset(iter);
3983	4580
3984	4581	again:
..	..	@@ -3986,14 +4583,13 @@
3986	4583	return NULL;
3987	4584
3988	4585	/*
3989		- * We repeat when a time extend is encountered or we hit
3990		- * the end of the page. Since the time extend is always attached
3991		- * to a data event, we should never loop more than three times.
3992		- * Once for going to next page, once on time extend, and
3993		- * finally once to get the event.
3994		- * (We never hit the following condition more than thrice).
	4586	+ * As the writer can mess with what the iterator is trying
	4587	+ * to read, just give up if we fail to get an event after
	4588	+ * three tries. The iterator is not as reliable when reading
	4589	+ * the ring buffer with an active write as the consumer is.
	4590	+ * Do not warn if the three failures is reached.
3995	4591	*/
3996		- if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3))
	4592	+ if (++nr_loops > 3)
3997	4593	return NULL;
3998	4594
3999	4595	if (rb_per_cpu_empty(cpu_buffer))
..	..	@@ -4005,6 +4601,8 @@
4005	4601	}
4006	4602
4007	4603	event = rb_iter_head_event(iter);
	4604	+ if (!event)
	4605	+ goto again;
4008	4606
4009	4607	switch (event->type_len) {
4010	4608	case RINGBUF_TYPE_PADDING:
..	..	@@ -4039,7 +4637,7 @@
4039	4637	return event;
4040	4638
4041	4639	default:
4042		- BUG();
	4640	+ RB_WARN_ON(cpu_buffer, 1);
4043	4641	}
4044	4642
4045	4643	return NULL;
..	..	@@ -4089,7 +4687,7 @@
4089	4687	* not consume the data.
4090	4688	*/
4091	4689	struct ring_buffer_event *
4092		-ring_buffer_peek(struct ring_buffer buffer, int cpu, u64 ts,
	4690	+ring_buffer_peek(struct trace_buffer buffer, int cpu, u64 ts,
4093	4691	unsigned long *lost_events)
4094	4692	{
4095	4693	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
..	..	@@ -4114,6 +4712,20 @@
4114	4712
4115	4713	return event;
4116	4714	}
	4715	+
	4716	+/** ring_buffer_iter_dropped - report if there are dropped events
	4717	+ * @iter: The ring buffer iterator
	4718	+ *
	4719	+ * Returns true if there was dropped events since the last peek.
	4720	+ */
	4721	+bool ring_buffer_iter_dropped(struct ring_buffer_iter *iter)
	4722	+{
	4723	+ bool ret = iter->missed_events != 0;
	4724	+
	4725	+ iter->missed_events = 0;
	4726	+ return ret;
	4727	+}
	4728	+EXPORT_SYMBOL_GPL(ring_buffer_iter_dropped);
4117	4729
4118	4730	/**
4119	4731	* ring_buffer_iter_peek - peek at the next event to be read
..	..	@@ -4153,7 +4765,7 @@
4153	4765	* and eventually empty the ring buffer if the producer is slower.
4154	4766	*/
4155	4767	struct ring_buffer_event *
4156		-ring_buffer_consume(struct ring_buffer buffer, int cpu, u64 ts,
	4768	+ring_buffer_consume(struct trace_buffer buffer, int cpu, u64 ts,
4157	4769	unsigned long *lost_events)
4158	4770	{
4159	4771	struct ring_buffer_per_cpu *cpu_buffer;
..	..	@@ -4213,7 +4825,7 @@
4213	4825	* This overall must be paired with ring_buffer_read_finish.
4214	4826	*/
4215	4827	struct ring_buffer_iter *
4216		-ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu, gfp_t flags)
	4828	+ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags)
4217	4829	{
4218	4830	struct ring_buffer_per_cpu *cpu_buffer;
4219	4831	struct ring_buffer_iter *iter;
..	..	@@ -4221,16 +4833,21 @@
4221	4833	if (!cpumask_test_cpu(cpu, buffer->cpumask))
4222	4834	return NULL;
4223	4835
4224		- iter = kmalloc(sizeof(*iter), flags);
	4836	+ iter = kzalloc(sizeof(*iter), flags);
4225	4837	if (!iter)
4226	4838	return NULL;
	4839	+
	4840	+ iter->event = kmalloc(BUF_MAX_DATA_SIZE, flags);
	4841	+ if (!iter->event) {
	4842	+ kfree(iter);
	4843	+ return NULL;
	4844	+ }
4227	4845
4228	4846	cpu_buffer = buffer->buffers[cpu];
4229	4847
4230	4848	iter->cpu_buffer = cpu_buffer;
4231	4849
4232		- atomic_inc(&buffer->resize_disabled);
4233		- atomic_inc(&cpu_buffer->record_disabled);
	4850	+ atomic_inc(&cpu_buffer->resize_disabled);
4234	4851
4235	4852	return iter;
4236	4853	}
..	..	@@ -4246,7 +4863,7 @@
4246	4863	void
4247	4864	ring_buffer_read_prepare_sync(void)
4248	4865	{
4249		- synchronize_sched();
	4866	+ synchronize_rcu();
4250	4867	}
4251	4868	EXPORT_SYMBOL_GPL(ring_buffer_read_prepare_sync);
4252	4869
..	..	@@ -4303,48 +4920,38 @@
4303	4920	rb_check_pages(cpu_buffer);
4304	4921	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4305	4922
4306		- atomic_dec(&cpu_buffer->record_disabled);
4307		- atomic_dec(&cpu_buffer->buffer->resize_disabled);
	4923	+ atomic_dec(&cpu_buffer->resize_disabled);
	4924	+ kfree(iter->event);
4308	4925	kfree(iter);
4309	4926	}
4310	4927	EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
4311	4928
4312	4929	/**
4313		- * ring_buffer_read - read the next item in the ring buffer by the iterator
	4930	+ * ring_buffer_iter_advance - advance the iterator to the next location
4314	4931	* @iter: The ring buffer iterator
4315		- * @ts: The time stamp of the event read.
4316	4932	*
4317		- * This reads the next event in the ring buffer and increments the iterator.
	4933	+ * Move the location of the iterator such that the next read will
	4934	+ * be the next location of the iterator.
4318	4935	*/
4319		-struct ring_buffer_event *
4320		-ring_buffer_read(struct ring_buffer_iter iter, u64 ts)
	4936	+void ring_buffer_iter_advance(struct ring_buffer_iter *iter)
4321	4937	{
4322		- struct ring_buffer_event *event;
4323	4938	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
4324	4939	unsigned long flags;
4325	4940
4326	4941	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
4327		- again:
4328		- event = rb_iter_peek(iter, ts);
4329		- if (!event)
4330		- goto out;
4331		-
4332		- if (event->type_len == RINGBUF_TYPE_PADDING)
4333		- goto again;
4334	4942
4335	4943	rb_advance_iter(iter);
4336		- out:
4337		- raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4338	4944
4339		- return event;
	4945	+ raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4340	4946	}
4341		-EXPORT_SYMBOL_GPL(ring_buffer_read);
	4947	+EXPORT_SYMBOL_GPL(ring_buffer_iter_advance);
4342	4948
4343	4949	/**
4344	4950	* ring_buffer_size - return the size of the ring buffer (in bytes)
4345	4951	* @buffer: The ring buffer.
	4952	+ * @cpu: The CPU to get ring buffer size from.
4346	4953	*/
4347		-unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu)
	4954	+unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu)
4348	4955	{
4349	4956	/*
4350	4957	* Earlier, this method returned
..	..	@@ -4359,28 +4966,34 @@
4359	4966	}
4360	4967	EXPORT_SYMBOL_GPL(ring_buffer_size);
4361	4968
	4969	+static void rb_clear_buffer_page(struct buffer_page *page)
	4970	+{
	4971	+ local_set(&page->write, 0);
	4972	+ local_set(&page->entries, 0);
	4973	+ rb_init_page(page->page);
	4974	+ page->read = 0;
	4975	+}
	4976	+
4362	4977	static void
4363	4978	rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
4364	4979	{
	4980	+ struct buffer_page *page;
	4981	+
4365	4982	rb_head_page_deactivate(cpu_buffer);
4366	4983
4367	4984	cpu_buffer->head_page
4368	4985	= list_entry(cpu_buffer->pages, struct buffer_page, list);
4369		- local_set(&cpu_buffer->head_page->write, 0);
4370		- local_set(&cpu_buffer->head_page->entries, 0);
4371		- local_set(&cpu_buffer->head_page->page->commit, 0);
4372		-
4373		- cpu_buffer->head_page->read = 0;
	4986	+ rb_clear_buffer_page(cpu_buffer->head_page);
	4987	+ list_for_each_entry(page, cpu_buffer->pages, list) {
	4988	+ rb_clear_buffer_page(page);
	4989	+ }
4374	4990
4375	4991	cpu_buffer->tail_page = cpu_buffer->head_page;
4376	4992	cpu_buffer->commit_page = cpu_buffer->head_page;
4377	4993
4378	4994	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
4379	4995	INIT_LIST_HEAD(&cpu_buffer->new_pages);
4380		- local_set(&cpu_buffer->reader_page->write, 0);
4381		- local_set(&cpu_buffer->reader_page->entries, 0);
4382		- local_set(&cpu_buffer->reader_page->page->commit, 0);
4383		- cpu_buffer->reader_page->read = 0;
	4996	+ rb_clear_buffer_page(cpu_buffer->reader_page);
4384	4997
4385	4998	local_set(&cpu_buffer->entries_bytes, 0);
4386	4999	local_set(&cpu_buffer->overrun, 0);
..	..	@@ -4389,38 +5002,28 @@
4389	5002	local_set(&cpu_buffer->entries, 0);
4390	5003	local_set(&cpu_buffer->committing, 0);
4391	5004	local_set(&cpu_buffer->commits, 0);
	5005	+ local_set(&cpu_buffer->pages_touched, 0);
	5006	+ local_set(&cpu_buffer->pages_lost, 0);
	5007	+ local_set(&cpu_buffer->pages_read, 0);
	5008	+ cpu_buffer->last_pages_touch = 0;
	5009	+ cpu_buffer->shortest_full = 0;
4392	5010	cpu_buffer->read = 0;
4393	5011	cpu_buffer->read_bytes = 0;
4394	5012
4395		- cpu_buffer->write_stamp = 0;
4396		- cpu_buffer->read_stamp = 0;
	5013	+ rb_time_set(&cpu_buffer->write_stamp, 0);
	5014	+ rb_time_set(&cpu_buffer->before_stamp, 0);
4397	5015
4398	5016	cpu_buffer->lost_events = 0;
4399	5017	cpu_buffer->last_overrun = 0;
4400	5018
4401	5019	rb_head_page_activate(cpu_buffer);
	5020	+ cpu_buffer->pages_removed = 0;
4402	5021	}
4403	5022
4404		-/**
4405		- * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
4406		- * @buffer: The ring buffer to reset a per cpu buffer of
4407		- * @cpu: The CPU buffer to be reset
4408		- */
4409		-void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
	5023	+/* Must have disabled the cpu buffer then done a synchronize_rcu */
	5024	+static void reset_disabled_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
4410	5025	{
4411		- struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
4412	5026	unsigned long flags;
4413		-
4414		- if (!cpumask_test_cpu(cpu, buffer->cpumask))
4415		- return;
4416		- /* prevent another thread from changing buffer sizes */
4417		- mutex_lock(&buffer->mutex);
4418		-
4419		- atomic_inc(&buffer->resize_disabled);
4420		- atomic_inc(&cpu_buffer->record_disabled);
4421		-
4422		- /* Make sure all commits have finished */
4423		- synchronize_sched();
4424	5027
4425	5028	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
4426	5029
..	..	@@ -4435,24 +5038,115 @@
4435	5038
4436	5039	out:
4437	5040	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
	5041	+}
	5042	+
	5043	+/**
	5044	+ * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
	5045	+ * @buffer: The ring buffer to reset a per cpu buffer of
	5046	+ * @cpu: The CPU buffer to be reset
	5047	+ */
	5048	+void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu)
	5049	+{
	5050	+ struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	5051	+
	5052	+ if (!cpumask_test_cpu(cpu, buffer->cpumask))
	5053	+ return;
	5054	+
	5055	+ /* prevent another thread from changing buffer sizes */
	5056	+ mutex_lock(&buffer->mutex);
	5057	+
	5058	+ atomic_inc(&cpu_buffer->resize_disabled);
	5059	+ atomic_inc(&cpu_buffer->record_disabled);
	5060	+
	5061	+ /* Make sure all commits have finished */
	5062	+ synchronize_rcu();
	5063	+
	5064	+ reset_disabled_cpu_buffer(cpu_buffer);
4438	5065
4439	5066	atomic_dec(&cpu_buffer->record_disabled);
4440		- atomic_dec(&buffer->resize_disabled);
	5067	+ atomic_dec(&cpu_buffer->resize_disabled);
4441	5068
4442	5069	mutex_unlock(&buffer->mutex);
4443	5070	}
4444	5071	EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
4445	5072
	5073	+/* Flag to ensure proper resetting of atomic variables */
	5074	+#define RESET_BIT (1 << 30)
	5075	+
	5076	+/**
	5077	+ * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
	5078	+ * @buffer: The ring buffer to reset a per cpu buffer of
	5079	+ * @cpu: The CPU buffer to be reset
	5080	+ */
	5081	+void ring_buffer_reset_online_cpus(struct trace_buffer *buffer)
	5082	+{
	5083	+ struct ring_buffer_per_cpu *cpu_buffer;
	5084	+ int cpu;
	5085	+
	5086	+ /* prevent another thread from changing buffer sizes */
	5087	+ mutex_lock(&buffer->mutex);
	5088	+
	5089	+ for_each_online_buffer_cpu(buffer, cpu) {
	5090	+ cpu_buffer = buffer->buffers[cpu];
	5091	+
	5092	+ atomic_add(RESET_BIT, &cpu_buffer->resize_disabled);
	5093	+ atomic_inc(&cpu_buffer->record_disabled);
	5094	+ }
	5095	+
	5096	+ /* Make sure all commits have finished */
	5097	+ synchronize_rcu();
	5098	+
	5099	+ for_each_buffer_cpu(buffer, cpu) {
	5100	+ cpu_buffer = buffer->buffers[cpu];
	5101	+
	5102	+ /*
	5103	+ * If a CPU came online during the synchronize_rcu(), then
	5104	+ * ignore it.
	5105	+ */
	5106	+ if (!(atomic_read(&cpu_buffer->resize_disabled) & RESET_BIT))
	5107	+ continue;
	5108	+
	5109	+ reset_disabled_cpu_buffer(cpu_buffer);
	5110	+
	5111	+ atomic_dec(&cpu_buffer->record_disabled);
	5112	+ atomic_sub(RESET_BIT, &cpu_buffer->resize_disabled);
	5113	+ }
	5114	+
	5115	+ mutex_unlock(&buffer->mutex);
	5116	+}
	5117	+
4446	5118	/**
4447	5119	* ring_buffer_reset - reset a ring buffer
4448	5120	* @buffer: The ring buffer to reset all cpu buffers
4449	5121	*/
4450		-void ring_buffer_reset(struct ring_buffer *buffer)
	5122	+void ring_buffer_reset(struct trace_buffer *buffer)
4451	5123	{
	5124	+ struct ring_buffer_per_cpu *cpu_buffer;
4452	5125	int cpu;
4453	5126
4454		- for_each_buffer_cpu(buffer, cpu)
4455		- ring_buffer_reset_cpu(buffer, cpu);
	5127	+ /* prevent another thread from changing buffer sizes */
	5128	+ mutex_lock(&buffer->mutex);
	5129	+
	5130	+ for_each_buffer_cpu(buffer, cpu) {
	5131	+ cpu_buffer = buffer->buffers[cpu];
	5132	+
	5133	+ atomic_inc(&cpu_buffer->resize_disabled);
	5134	+ atomic_inc(&cpu_buffer->record_disabled);
	5135	+ }
	5136	+
	5137	+ /* Make sure all commits have finished */
	5138	+ synchronize_rcu();
	5139	+
	5140	+ for_each_buffer_cpu(buffer, cpu) {
	5141	+ cpu_buffer = buffer->buffers[cpu];
	5142	+
	5143	+ reset_disabled_cpu_buffer(cpu_buffer);
	5144	+
	5145	+ atomic_dec(&cpu_buffer->record_disabled);
	5146	+ atomic_dec(&cpu_buffer->resize_disabled);
	5147	+ }
	5148	+
	5149	+ mutex_unlock(&buffer->mutex);
4456	5150	}
4457	5151	EXPORT_SYMBOL_GPL(ring_buffer_reset);
4458	5152
..	..	@@ -4460,7 +5154,7 @@
4460	5154	* rind_buffer_empty - is the ring buffer empty?
4461	5155	* @buffer: The ring buffer to test
4462	5156	*/
4463		-bool ring_buffer_empty(struct ring_buffer *buffer)
	5157	+bool ring_buffer_empty(struct trace_buffer *buffer)
4464	5158	{
4465	5159	struct ring_buffer_per_cpu *cpu_buffer;
4466	5160	unsigned long flags;
..	..	@@ -4490,7 +5184,7 @@
4490	5184	* @buffer: The ring buffer
4491	5185	* @cpu: The CPU buffer to test
4492	5186	*/
4493		-bool ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
	5187	+bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu)
4494	5188	{
4495	5189	struct ring_buffer_per_cpu *cpu_buffer;
4496	5190	unsigned long flags;
..	..	@@ -4516,14 +5210,15 @@
4516	5210	* ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
4517	5211	* @buffer_a: One buffer to swap with
4518	5212	* @buffer_b: The other buffer to swap with
	5213	+ * @cpu: the CPU of the buffers to swap
4519	5214	*
4520	5215	* This function is useful for tracers that want to take a "snapshot"
4521	5216	* of a CPU buffer and has another back up buffer lying around.
4522	5217	* it is expected that the tracer handles the cpu buffer not being
4523	5218	* used at the moment.
4524	5219	*/
4525		-int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
4526		- struct ring_buffer *buffer_b, int cpu)
	5220	+int ring_buffer_swap_cpu(struct trace_buffer *buffer_a,
	5221	+ struct trace_buffer *buffer_b, int cpu)
4527	5222	{
4528	5223	struct ring_buffer_per_cpu *cpu_buffer_a;
4529	5224	struct ring_buffer_per_cpu *cpu_buffer_b;
..	..	@@ -4555,7 +5250,7 @@
4555	5250	goto out;
4556	5251
4557	5252	/*
4558		- * We can't do a synchronize_sched here because this
	5253	+ * We can't do a synchronize_rcu here because this
4559	5254	* function can be called in atomic context.
4560	5255	* Normally this will be called from the same CPU as cpu.
4561	5256	* If not it's up to the caller to protect this.
..	..	@@ -4567,6 +5262,15 @@
4567	5262	if (local_read(&cpu_buffer_a->committing))
4568	5263	goto out_dec;
4569	5264	if (local_read(&cpu_buffer_b->committing))
	5265	+ goto out_dec;
	5266	+
	5267	+ /*
	5268	+ * When resize is in progress, we cannot swap it because
	5269	+ * it will mess the state of the cpu buffer.
	5270	+ */
	5271	+ if (atomic_read(&buffer_a->resizing))
	5272	+ goto out_dec;
	5273	+ if (atomic_read(&buffer_b->resizing))
4570	5274	goto out_dec;
4571	5275
4572	5276	buffer_a->buffers[cpu] = cpu_buffer_b;
..	..	@@ -4602,7 +5306,7 @@
4602	5306	* Returns:
4603	5307	* The page allocated, or ERR_PTR
4604	5308	*/
4605		-void ring_buffer_alloc_read_page(struct ring_buffer buffer, int cpu)
	5309	+void ring_buffer_alloc_read_page(struct trace_buffer buffer, int cpu)
4606	5310	{
4607	5311	struct ring_buffer_per_cpu *cpu_buffer;
4608	5312	struct buffer_data_page *bpage = NULL;
..	..	@@ -4649,12 +5353,17 @@
4649	5353	*
4650	5354	* Free a page allocated from ring_buffer_alloc_read_page.
4651	5355	*/
4652		-void ring_buffer_free_read_page(struct ring_buffer buffer, int cpu, void data)
	5356	+void ring_buffer_free_read_page(struct trace_buffer buffer, int cpu, void data)
4653	5357	{
4654		- struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	5358	+ struct ring_buffer_per_cpu *cpu_buffer;
4655	5359	struct buffer_data_page *bpage = data;
4656	5360	struct page *page = virt_to_page(bpage);
4657	5361	unsigned long flags;
	5362	+
	5363	+ if (!buffer \|\| !buffer->buffers \|\| !buffer->buffers[cpu])
	5364	+ return;
	5365	+
	5366	+ cpu_buffer = buffer->buffers[cpu];
4658	5367
4659	5368	/* If the page is still in use someplace else, we can't reuse it */
4660	5369	if (page_ref_count(page) > 1)
..	..	@@ -4709,7 +5418,7 @@
4709	5418	* >=0 if data has been transferred, returns the offset of consumed data.
4710	5419	* <0 if no data has been transferred.
4711	5420	*/
4712		-int ring_buffer_read_page(struct ring_buffer *buffer,
	5421	+int ring_buffer_read_page(struct trace_buffer *buffer,
4713	5422	void **data_page, size_t len, int cpu, int full)
4714	5423	{
4715	5424	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
..	..	@@ -4770,7 +5479,15 @@
4770	5479	unsigned int pos = 0;
4771	5480	unsigned int size;
4772	5481
4773		- if (full)
	5482	+ /*
	5483	+ * If a full page is expected, this can still be returned
	5484	+ * if there's been a previous partial read and the
	5485	+ * rest of the page can be read and the commit page is off
	5486	+ * the reader page.
	5487	+ */
	5488	+ if (full &&
	5489	+ (!read \|\| (len < (commit - read)) \|\|
	5490	+ cpu_buffer->reader_page == cpu_buffer->commit_page))
4774	5491	goto out_unlock;
4775	5492
4776	5493	if (len > (commit - read))
..	..	@@ -4819,7 +5536,7 @@
4819	5536	} else {
4820	5537	/* update the entry counter */
4821	5538	cpu_buffer->read += rb_page_entries(reader);
4822		- cpu_buffer->read_bytes += BUF_PAGE_SIZE;
	5539	+ cpu_buffer->read_bytes += rb_page_commit(reader);
4823	5540
4824	5541	/* swap the pages */
4825	5542	rb_init_page(bpage);
..	..	@@ -4880,12 +5597,12 @@
4880	5597	*/
4881	5598	int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node)
4882	5599	{
4883		- struct ring_buffer *buffer;
	5600	+ struct trace_buffer *buffer;
4884	5601	long nr_pages_same;
4885	5602	int cpu_i;
4886	5603	unsigned long nr_pages;
4887	5604
4888		- buffer = container_of(node, struct ring_buffer, node);
	5605	+ buffer = container_of(node, struct trace_buffer, node);
4889	5606	if (cpumask_test_cpu(cpu, buffer->cpumask))
4890	5607	return 0;
4891	5608
..	..	@@ -4935,7 +5652,7 @@
4935	5652	static struct task_struct *rb_threads[NR_CPUS] __initdata;
4936	5653
4937	5654	struct rb_test_data {
4938		- struct ring_buffer *buffer;
	5655	+ struct trace_buffer *buffer;
4939	5656	unsigned long events;
4940	5657	unsigned long bytes_written;
4941	5658	unsigned long bytes_alloc;
..	..	@@ -4983,7 +5700,7 @@
4983	5700	cnt = data->cnt + (nested ? 27 : 0);
4984	5701
4985	5702	/* Multiply cnt by ~e, to make some unique increment */
4986		- size = (data->cnt * 68 / 25) % (sizeof(rb_string) - 1);
	5703	+ size = (cnt * 68 / 25) % (sizeof(rb_string) - 1);
4987	5704
4988	5705	len = size + sizeof(struct rb_item);
4989	5706
..	..	@@ -5077,10 +5794,15 @@
5077	5794	static __init int test_ringbuffer(void)
5078	5795	{
5079	5796	struct task_struct *rb_hammer;
5080		- struct ring_buffer *buffer;
	5797	+ struct trace_buffer *buffer;
5081	5798	int cpu;
5082	5799	int ret = 0;
5083	5800
	5801	+ if (security_locked_down(LOCKDOWN_TRACEFS)) {
	5802	+ pr_warn("Lockdown is enabled, skipping ring buffer tests\n");
	5803	+ return 0;
	5804	+ }
	5805	+
5084	5806	pr_info("Running ring buffer tests...\n");
5085	5807
5086	5808	buffer = ring_buffer_alloc(RB_TEST_BUFFER_SIZE, RB_FL_OVERWRITE);