~hc/RK356X_SDK_RELEASE.git

..	..	@@ -14,6 +14,7 @@
14	14	#include <linux/fs.h>
15	15	#include <linux/log2.h>
16	16	#include <linux/mount.h>
	17	+#include <linux/pseudo_fs.h>
17	18	#include <linux/magic.h>
18	19	#include <linux/pipe_fs_i.h>
19	20	#include <linux/uio.h>
..	..	@@ -23,6 +24,7 @@
23	24	#include <linux/syscalls.h>
24	25	#include <linux/fcntl.h>
25	26	#include <linux/memcontrol.h>
	27	+#include <linux/watch_queue.h>
26	28
27	29	#include <linux/uaccess.h>
28	30	#include <asm/ioctls.h>
..	..	@@ -57,10 +59,12 @@
57	59	unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;
58	60
59	61	/*
60		- * We use a start+len construction, which provides full use of the
61		- * allocated memory.
62		- * -- Florian Coosmann (FGC)
63		- *
	62	+ * We use head and tail indices that aren't masked off, except at the point of
	63	+ * dereference, but rather they're allowed to wrap naturally. This means there
	64	+ * isn't a dead spot in the buffer, but the ring has to be a power of two and
	65	+ * <= 2^31.
	66	+ * -- David Howells 2019-09-23.
	67	+ *
64	68	* Reads with count = 0 should always return 0.
65	69	* -- Julian Bradfield 1999-06-07.
66	70	*
..	..	@@ -117,22 +121,6 @@
117	121	}
118	122	}
119	123
120		-/* Drop the inode semaphore and wait for a pipe event, atomically */
121		-void pipe_wait(struct pipe_inode_info *pipe)
122		-{
123		- DEFINE_WAIT(wait);
124		-
125		- /*
126		- * Pipes are system-local resources, so sleeping on them
127		- * is considered a noninteractive wait:
128		- */
129		- prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE);
130		- pipe_unlock(pipe);
131		- schedule();
132		- finish_wait(&pipe->wait, &wait);
133		- pipe_lock(pipe);
134		-}
135		-
136	124	static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
137	125	struct pipe_buffer *buf)
138	126	{
..	..	@@ -149,22 +137,20 @@
149	137	put_page(page);
150	138	}
151	139
152		-static int anon_pipe_buf_steal(struct pipe_inode_info *pipe,
153		- struct pipe_buffer *buf)
	140	+static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
	141	+ struct pipe_buffer *buf)
154	142	{
155	143	struct page *page = buf->page;
156	144
157		- if (page_count(page) == 1) {
158		- if (memcg_kmem_enabled())
159		- memcg_kmem_uncharge(page, 0);
160		- __SetPageLocked(page);
161		- return 0;
162		- }
163		- return 1;
	145	+ if (page_count(page) != 1)
	146	+ return false;
	147	+ memcg_kmem_uncharge_page(page, 0);
	148	+ __SetPageLocked(page);
	149	+ return true;
164	150	}
165	151
166	152	/**
167		- * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer
	153	+ * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer
168	154	* @pipe: the pipe that the buffer belongs to
169	155	* @buf: the buffer to attempt to steal
170	156	*
..	..	@@ -175,8 +161,8 @@
175	161	* he wishes; the typical use is insertion into a different file
176	162	* page cache.
177	163	*/
178		-int generic_pipe_buf_steal(struct pipe_inode_info *pipe,
179		- struct pipe_buffer *buf)
	164	+bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,
	165	+ struct pipe_buffer *buf)
180	166	{
181	167	struct page *page = buf->page;
182	168
..	..	@@ -187,12 +173,11 @@
187	173	*/
188	174	if (page_count(page) == 1) {
189	175	lock_page(page);
190		- return 0;
	176	+ return true;
191	177	}
192		-
193		- return 1;
	178	+ return false;
194	179	}
195		-EXPORT_SYMBOL(generic_pipe_buf_steal);
	180	+EXPORT_SYMBOL(generic_pipe_buf_try_steal);
196	181
197	182	/**
198	183	* generic_pipe_buf_get - get a reference to a &struct pipe_buffer
..	..	@@ -211,22 +196,6 @@
211	196	EXPORT_SYMBOL(generic_pipe_buf_get);
212	197
213	198	/**
214		- * generic_pipe_buf_confirm - verify contents of the pipe buffer
215		- * @info: the pipe that the buffer belongs to
216		- * @buf: the buffer to confirm
217		- *
218		- * Description:
219		- * This function does nothing, because the generic pipe code uses
220		- * pages that are always good when inserted into the pipe.
221		- */
222		-int generic_pipe_buf_confirm(struct pipe_inode_info *info,
223		- struct pipe_buffer *buf)
224		-{
225		- return 0;
226		-}
227		-EXPORT_SYMBOL(generic_pipe_buf_confirm);
228		-
229		-/**
230	199	* generic_pipe_buf_release - put a reference to a &struct pipe_buffer
231	200	* @pipe: the pipe that the buffer belongs to
232	201	* @buf: the buffer to put a reference to
..	..	@@ -242,33 +211,19 @@
242	211	EXPORT_SYMBOL(generic_pipe_buf_release);
243	212
244	213	static const struct pipe_buf_operations anon_pipe_buf_ops = {
245		- .can_merge = 1,
246		- .confirm = generic_pipe_buf_confirm,
247		- .release = anon_pipe_buf_release,
248		- .steal = anon_pipe_buf_steal,
249		- .get = generic_pipe_buf_get,
	214	+ .release = anon_pipe_buf_release,
	215	+ .try_steal = anon_pipe_buf_try_steal,
	216	+ .get = generic_pipe_buf_get,
250	217	};
251	218
252		-static const struct pipe_buf_operations anon_pipe_buf_nomerge_ops = {
253		- .can_merge = 0,
254		- .confirm = generic_pipe_buf_confirm,
255		- .release = anon_pipe_buf_release,
256		- .steal = anon_pipe_buf_steal,
257		- .get = generic_pipe_buf_get,
258		-};
259		-
260		-static const struct pipe_buf_operations packet_pipe_buf_ops = {
261		- .can_merge = 0,
262		- .confirm = generic_pipe_buf_confirm,
263		- .release = anon_pipe_buf_release,
264		- .steal = anon_pipe_buf_steal,
265		- .get = generic_pipe_buf_get,
266		-};
267		-
268		-void pipe_buf_mark_unmergeable(struct pipe_buffer *buf)
	219	+/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
	220	+static inline bool pipe_readable(const struct pipe_inode_info *pipe)
269	221	{
270		- if (buf->ops == &anon_pipe_buf_ops)
271		- buf->ops = &anon_pipe_buf_nomerge_ops;
	222	+ unsigned int head = READ_ONCE(pipe->head);
	223	+ unsigned int tail = READ_ONCE(pipe->tail);
	224	+ unsigned int writers = READ_ONCE(pipe->writers);
	225	+
	226	+ return !pipe_empty(head, tail) \|\| !writers;
272	227	}
273	228
274	229	static ssize_t
..	..	@@ -277,27 +232,69 @@
277	232	size_t total_len = iov_iter_count(to);
278	233	struct file *filp = iocb->ki_filp;
279	234	struct pipe_inode_info *pipe = filp->private_data;
280		- int do_wakeup;
	235	+ bool was_full, wake_next_reader = false;
281	236	ssize_t ret;
282	237
283	238	/* Null read succeeds. */
284	239	if (unlikely(total_len == 0))
285	240	return 0;
286	241
287		- do_wakeup = 0;
288	242	ret = 0;
289	243	__pipe_lock(pipe);
	244	+
	245	+ /*
	246	+ * We only wake up writers if the pipe was full when we started
	247	+ * reading in order to avoid unnecessary wakeups.
	248	+ *
	249	+ * But when we do wake up writers, we do so using a sync wakeup
	250	+ * (WF_SYNC), because we want them to get going and generate more
	251	+ * data for us.
	252	+ */
	253	+ was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
290	254	for (;;) {
291		- int bufs = pipe->nrbufs;
292		- if (bufs) {
293		- int curbuf = pipe->curbuf;
294		- struct pipe_buffer *buf = pipe->bufs + curbuf;
	255	+ /* Read ->head with a barrier vs post_one_notification() */
	256	+ unsigned int head = smp_load_acquire(&pipe->head);
	257	+ unsigned int tail = pipe->tail;
	258	+ unsigned int mask = pipe->ring_size - 1;
	259	+
	260	+#ifdef CONFIG_WATCH_QUEUE
	261	+ if (pipe->note_loss) {
	262	+ struct watch_notification n;
	263	+
	264	+ if (total_len < 8) {
	265	+ if (ret == 0)
	266	+ ret = -ENOBUFS;
	267	+ break;
	268	+ }
	269	+
	270	+ n.type = WATCH_TYPE_META;
	271	+ n.subtype = WATCH_META_LOSS_NOTIFICATION;
	272	+ n.info = watch_sizeof(n);
	273	+ if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {
	274	+ if (ret == 0)
	275	+ ret = -EFAULT;
	276	+ break;
	277	+ }
	278	+ ret += sizeof(n);
	279	+ total_len -= sizeof(n);
	280	+ pipe->note_loss = false;
	281	+ }
	282	+#endif
	283	+
	284	+ if (!pipe_empty(head, tail)) {
	285	+ struct pipe_buffer *buf = &pipe->bufs[tail & mask];
295	286	size_t chars = buf->len;
296	287	size_t written;
297	288	int error;
298	289
299		- if (chars > total_len)
	290	+ if (chars > total_len) {
	291	+ if (buf->flags & PIPE_BUF_FLAG_WHOLE) {
	292	+ if (ret == 0)
	293	+ ret = -ENOBUFS;
	294	+ break;
	295	+ }
300	296	chars = total_len;
	297	+ }
301	298
302	299	error = pipe_buf_confirm(pipe, buf);
303	300	if (error) {
..	..	@@ -324,50 +321,77 @@
324	321
325	322	if (!buf->len) {
326	323	pipe_buf_release(pipe, buf);
327		- curbuf = (curbuf + 1) & (pipe->buffers - 1);
328		- pipe->curbuf = curbuf;
329		- pipe->nrbufs = --bufs;
330		- do_wakeup = 1;
	324	+ spin_lock_irq(&pipe->rd_wait.lock);
	325	+#ifdef CONFIG_WATCH_QUEUE
	326	+ if (buf->flags & PIPE_BUF_FLAG_LOSS)
	327	+ pipe->note_loss = true;
	328	+#endif
	329	+ tail++;
	330	+ pipe->tail = tail;
	331	+ spin_unlock_irq(&pipe->rd_wait.lock);
331	332	}
332	333	total_len -= chars;
333	334	if (!total_len)
334	335	break; /* common path: read succeeded */
	336	+ if (!pipe_empty(head, tail)) /* More to do? */
	337	+ continue;
335	338	}
336		- if (bufs) /* More to do? */
337		- continue;
	339	+
338	340	if (!pipe->writers)
339	341	break;
340		- if (!pipe->waiting_writers) {
341		- /* syscall merging: Usually we must not sleep
342		- * if O_NONBLOCK is set, or if we got some data.
343		- * But if a writer sleeps in kernel space, then
344		- * we can wait for that data without violating POSIX.
345		- */
346		- if (ret)
347		- break;
348		- if (filp->f_flags & O_NONBLOCK) {
349		- ret = -EAGAIN;
350		- break;
351		- }
352		- }
353		- if (signal_pending(current)) {
354		- if (!ret)
355		- ret = -ERESTARTSYS;
	342	+ if (ret)
	343	+ break;
	344	+ if (filp->f_flags & O_NONBLOCK) {
	345	+ ret = -EAGAIN;
356	346	break;
357	347	}
358		- if (do_wakeup) {
359		- wake_up_interruptible_sync_poll(&pipe->wait, EPOLLOUT \| EPOLLWRNORM);
360		- kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
	348	+ __pipe_unlock(pipe);
	349	+
	350	+ /*
	351	+ * We only get here if we didn't actually read anything.
	352	+ *
	353	+ * However, we could have seen (and removed) a zero-sized
	354	+ * pipe buffer, and might have made space in the buffers
	355	+ * that way.
	356	+ *
	357	+ * You can't make zero-sized pipe buffers by doing an empty
	358	+ * write (not even in packet mode), but they can happen if
	359	+ * the writer gets an EFAULT when trying to fill a buffer
	360	+ * that already got allocated and inserted in the buffer
	361	+ * array.
	362	+ *
	363	+ * So we still need to wake up any pending writers in the
	364	+ * _very_ unlikely case that the pipe was full, but we got
	365	+ * no data.
	366	+ */
	367	+ if (unlikely(was_full)) {
	368	+ wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT \| EPOLLWRNORM);
	369	+ kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
361	370	}
362		- pipe_wait(pipe);
	371	+
	372	+ /*
	373	+ * But because we didn't read anything, at this point we can
	374	+ * just return directly with -ERESTARTSYS if we're interrupted,
	375	+ * since we've done any required wakeups and there's no need
	376	+ * to mark anything accessed. And we've dropped the lock.
	377	+ */
	378	+ if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)
	379	+ return -ERESTARTSYS;
	380	+
	381	+ __pipe_lock(pipe);
	382	+ was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
	383	+ wake_next_reader = true;
363	384	}
	385	+ if (pipe_empty(pipe->head, pipe->tail))
	386	+ wake_next_reader = false;
364	387	__pipe_unlock(pipe);
365	388
366		- /* Signal writers asynchronously that there is more room. */
367		- if (do_wakeup) {
368		- wake_up_interruptible_sync_poll(&pipe->wait, EPOLLOUT \| EPOLLWRNORM);
	389	+ if (was_full) {
	390	+ wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT \| EPOLLWRNORM);
369	391	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
370	392	}
	393	+ if (wake_next_reader)
	394	+ wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN \| EPOLLRDNORM);
371	395	if (ret > 0)
372	396	file_accessed(filp);
373	397	return ret;
..	..	@@ -378,15 +402,28 @@
378	402	return (file->f_flags & O_DIRECT) != 0;
379	403	}
380	404
	405	+/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
	406	+static inline bool pipe_writable(const struct pipe_inode_info *pipe)
	407	+{
	408	+ unsigned int head = READ_ONCE(pipe->head);
	409	+ unsigned int tail = READ_ONCE(pipe->tail);
	410	+ unsigned int max_usage = READ_ONCE(pipe->max_usage);
	411	+
	412	+ return !pipe_full(head, tail, max_usage) \|\|
	413	+ !READ_ONCE(pipe->readers);
	414	+}
	415	+
381	416	static ssize_t
382	417	pipe_write(struct kiocb iocb, struct iov_iter from)
383	418	{
384	419	struct file *filp = iocb->ki_filp;
385	420	struct pipe_inode_info *pipe = filp->private_data;
	421	+ unsigned int head;
386	422	ssize_t ret = 0;
387		- int do_wakeup = 0;
388	423	size_t total_len = iov_iter_count(from);
389	424	ssize_t chars;
	425	+ bool was_empty = false;
	426	+ bool wake_next_writer = false;
390	427
391	428	/* Null write succeeds. */
392	429	if (unlikely(total_len == 0))
..	..	@@ -400,15 +437,34 @@
400	437	goto out;
401	438	}
402	439
403		- /* We try to merge small writes */
404		- chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */
405		- if (pipe->nrbufs && chars != 0) {
406		- int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) &
407		- (pipe->buffers - 1);
408		- struct pipe_buffer *buf = pipe->bufs + lastbuf;
	440	+#ifdef CONFIG_WATCH_QUEUE
	441	+ if (pipe->watch_queue) {
	442	+ ret = -EXDEV;
	443	+ goto out;
	444	+ }
	445	+#endif
	446	+
	447	+ /*
	448	+ * Epoll nonsensically wants a wakeup whether the pipe
	449	+ * was already empty or not.
	450	+ *
	451	+ * If it wasn't empty we try to merge new data into
	452	+ * the last buffer.
	453	+ *
	454	+ * That naturally merges small writes, but it also
	455	+ * page-aligns the rest of the writes for large writes
	456	+ * spanning multiple pages.
	457	+ */
	458	+ head = pipe->head;
	459	+ was_empty = true;
	460	+ chars = total_len & (PAGE_SIZE-1);
	461	+ if (chars && !pipe_empty(head, pipe->tail)) {
	462	+ unsigned int mask = pipe->ring_size - 1;
	463	+ struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];
409	464	int offset = buf->offset + buf->len;
410	465
411		- if (buf->ops->can_merge && offset + chars <= PAGE_SIZE) {
	466	+ if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&
	467	+ offset + chars <= PAGE_SIZE) {
412	468	ret = pipe_buf_confirm(pipe, buf);
413	469	if (ret)
414	470	goto out;
..	..	@@ -418,7 +474,7 @@
418	474	ret = -EFAULT;
419	475	goto out;
420	476	}
421		- do_wakeup = 1;
	477	+
422	478	buf->len += ret;
423	479	if (!iov_iter_count(from))
424	480	goto out;
..	..	@@ -426,18 +482,17 @@
426	482	}
427	483
428	484	for (;;) {
429		- int bufs;
430		-
431	485	if (!pipe->readers) {
432	486	send_sig(SIGPIPE, current, 0);
433	487	if (!ret)
434	488	ret = -EPIPE;
435	489	break;
436	490	}
437		- bufs = pipe->nrbufs;
438		- if (bufs < pipe->buffers) {
439		- int newbuf = (pipe->curbuf + bufs) & (pipe->buffers-1);
440		- struct pipe_buffer *buf = pipe->bufs + newbuf;
	491	+
	492	+ head = pipe->head;
	493	+ if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
	494	+ unsigned int mask = pipe->ring_size - 1;
	495	+ struct pipe_buffer *buf = &pipe->bufs[head & mask];
441	496	struct page *page = pipe->tmp_page;
442	497	int copied;
443	498
..	..	@@ -449,12 +504,35 @@
449	504	}
450	505	pipe->tmp_page = page;
451	506	}
452		- /* Always wake up, even if the copy fails. Otherwise
453		- * we lock up (O_NONBLOCK-)readers that sleep due to
454		- * syscall merging.
455		- * FIXME! Is this really true?
	507	+
	508	+ /* Allocate a slot in the ring in advance and attach an
	509	+ * empty buffer. If we fault or otherwise fail to use
	510	+ * it, either the reader will consume it or it'll still
	511	+ * be there for the next write.
456	512	*/
457		- do_wakeup = 1;
	513	+ spin_lock_irq(&pipe->rd_wait.lock);
	514	+
	515	+ head = pipe->head;
	516	+ if (pipe_full(head, pipe->tail, pipe->max_usage)) {
	517	+ spin_unlock_irq(&pipe->rd_wait.lock);
	518	+ continue;
	519	+ }
	520	+
	521	+ pipe->head = head + 1;
	522	+ spin_unlock_irq(&pipe->rd_wait.lock);
	523	+
	524	+ /* Insert it into the buffer array */
	525	+ buf = &pipe->bufs[head & mask];
	526	+ buf->page = page;
	527	+ buf->ops = &anon_pipe_buf_ops;
	528	+ buf->offset = 0;
	529	+ buf->len = 0;
	530	+ if (is_packetized(filp))
	531	+ buf->flags = PIPE_BUF_FLAG_PACKET;
	532	+ else
	533	+ buf->flags = PIPE_BUF_FLAG_CAN_MERGE;
	534	+ pipe->tmp_page = NULL;
	535	+
458	536	copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
459	537	if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
460	538	if (!ret)
..	..	@@ -462,25 +540,17 @@
462	540	break;
463	541	}
464	542	ret += copied;
465		-
466		- /* Insert it into the buffer array */
467		- buf->page = page;
468		- buf->ops = &anon_pipe_buf_ops;
469	543	buf->offset = 0;
470	544	buf->len = copied;
471		- buf->flags = 0;
472		- if (is_packetized(filp)) {
473		- buf->ops = &packet_pipe_buf_ops;
474		- buf->flags = PIPE_BUF_FLAG_PACKET;
475		- }
476		- pipe->nrbufs = ++bufs;
477		- pipe->tmp_page = NULL;
478	545
479	546	if (!iov_iter_count(from))
480	547	break;
481	548	}
482		- if (bufs < pipe->buffers)
	549	+
	550	+ if (!pipe_full(head, pipe->tail, pipe->max_usage))
483	551	continue;
	552	+
	553	+ /* Wait for buffer space to become available. */
484	554	if (filp->f_flags & O_NONBLOCK) {
485	555	if (!ret)
486	556	ret = -EAGAIN;
..	..	@@ -491,21 +561,43 @@
491	561	ret = -ERESTARTSYS;
492	562	break;
493	563	}
494		- if (do_wakeup) {
495		- wake_up_interruptible_sync_poll(&pipe->wait, EPOLLIN \| EPOLLRDNORM);
	564	+
	565	+ /*
	566	+ * We're going to release the pipe lock and wait for more
	567	+ * space. We wake up any readers if necessary, and then
	568	+ * after waiting we need to re-check whether the pipe
	569	+ * become empty while we dropped the lock.
	570	+ */
	571	+ __pipe_unlock(pipe);
	572	+ if (was_empty) {
	573	+ wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN \| EPOLLRDNORM);
496	574	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
497		- do_wakeup = 0;
498	575	}
499		- pipe->waiting_writers++;
500		- pipe_wait(pipe);
501		- pipe->waiting_writers--;
	576	+ wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));
	577	+ __pipe_lock(pipe);
	578	+ was_empty = pipe_empty(pipe->head, pipe->tail);
	579	+ wake_next_writer = true;
502	580	}
503	581	out:
	582	+ if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
	583	+ wake_next_writer = false;
504	584	__pipe_unlock(pipe);
505		- if (do_wakeup) {
506		- wake_up_interruptible_sync_poll(&pipe->wait, EPOLLIN \| EPOLLRDNORM);
	585	+
	586	+ /*
	587	+ * If we do do a wakeup event, we do a 'sync' wakeup, because we
	588	+ * want the reader to start processing things asap, rather than
	589	+ * leave the data pending.
	590	+ *
	591	+ * This is particularly important for small writes, because of
	592	+ * how (for example) the GNU make jobserver uses small writes to
	593	+ * wake up pending jobs
	594	+ */
	595	+ if (was_empty) {
	596	+ wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN \| EPOLLRDNORM);
507	597	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
508	598	}
	599	+ if (wake_next_writer)
	600	+ wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT \| EPOLLWRNORM);
509	601	if (ret > 0 && sb_start_write_trylock(file_inode(filp)->i_sb)) {
510	602	int err = file_update_time(filp);
511	603	if (err)
..	..	@@ -518,23 +610,40 @@
518	610	static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
519	611	{
520	612	struct pipe_inode_info *pipe = filp->private_data;
521		- int count, buf, nrbufs;
	613	+ int count, head, tail, mask;
522	614
523	615	switch (cmd) {
524		- case FIONREAD:
525		- __pipe_lock(pipe);
526		- count = 0;
527		- buf = pipe->curbuf;
528		- nrbufs = pipe->nrbufs;
529		- while (--nrbufs >= 0) {
530		- count += pipe->bufs[buf].len;
531		- buf = (buf+1) & (pipe->buffers - 1);
532		- }
533		- __pipe_unlock(pipe);
	616	+ case FIONREAD:
	617	+ __pipe_lock(pipe);
	618	+ count = 0;
	619	+ head = pipe->head;
	620	+ tail = pipe->tail;
	621	+ mask = pipe->ring_size - 1;
534	622
535		- return put_user(count, (int __user *)arg);
536		- default:
537		- return -ENOIOCTLCMD;
	623	+ while (tail != head) {
	624	+ count += pipe->bufs[tail & mask].len;
	625	+ tail++;
	626	+ }
	627	+ __pipe_unlock(pipe);
	628	+
	629	+ return put_user(count, (int __user *)arg);
	630	+
	631	+#ifdef CONFIG_WATCH_QUEUE
	632	+ case IOC_WATCH_QUEUE_SET_SIZE: {
	633	+ int ret;
	634	+ __pipe_lock(pipe);
	635	+ ret = watch_queue_set_size(pipe, arg);
	636	+ __pipe_unlock(pipe);
	637	+ return ret;
	638	+ }
	639	+
	640	+ case IOC_WATCH_QUEUE_SET_FILTER:
	641	+ return watch_queue_set_filter(
	642	+ pipe, (struct watch_notification_filter __user *)arg);
	643	+#endif
	644	+
	645	+ default:
	646	+ return -ENOIOCTLCMD;
538	647	}
539	648	}
540	649
..	..	@@ -544,21 +653,38 @@
544	653	{
545	654	__poll_t mask;
546	655	struct pipe_inode_info *pipe = filp->private_data;
547		- int nrbufs;
	656	+ unsigned int head, tail;
548	657
549		- poll_wait(filp, &pipe->wait, wait);
	658	+ /*
	659	+ * Reading pipe state only -- no need for acquiring the semaphore.
	660	+ *
	661	+ * But because this is racy, the code has to add the
	662	+ * entry to the poll table _first_ ..
	663	+ */
	664	+ if (filp->f_mode & FMODE_READ)
	665	+ poll_wait(filp, &pipe->rd_wait, wait);
	666	+ if (filp->f_mode & FMODE_WRITE)
	667	+ poll_wait(filp, &pipe->wr_wait, wait);
550	668
551		- /* Reading only -- no need for acquiring the semaphore. */
552		- nrbufs = pipe->nrbufs;
	669	+ /*
	670	+ * .. and only then can you do the racy tests. That way,
	671	+ * if something changes and you got it wrong, the poll
	672	+ * table entry will wake you up and fix it.
	673	+ */
	674	+ head = READ_ONCE(pipe->head);
	675	+ tail = READ_ONCE(pipe->tail);
	676	+
553	677	mask = 0;
554	678	if (filp->f_mode & FMODE_READ) {
555		- mask = (nrbufs > 0) ? EPOLLIN \| EPOLLRDNORM : 0;
	679	+ if (!pipe_empty(head, tail))
	680	+ mask \|= EPOLLIN \| EPOLLRDNORM;
556	681	if (!pipe->writers && filp->f_version != pipe->w_counter)
557	682	mask \|= EPOLLHUP;
558	683	}
559	684
560	685	if (filp->f_mode & FMODE_WRITE) {
561		- mask \|= (nrbufs < pipe->buffers) ? EPOLLOUT \| EPOLLWRNORM : 0;
	686	+ if (!pipe_full(head, tail, pipe->max_usage))
	687	+ mask \|= EPOLLOUT \| EPOLLWRNORM;
562	688	/*
563	689	* Most Unices do not set EPOLLERR for FIFOs but on Linux they
564	690	* behave exactly like pipes for poll().
..	..	@@ -596,8 +722,10 @@
596	722	if (file->f_mode & FMODE_WRITE)
597	723	pipe->writers--;
598	724
599		- if (pipe->readers \|\| pipe->writers) {
600		- wake_up_interruptible_sync_poll(&pipe->wait, EPOLLIN \| EPOLLOUT \| EPOLLRDNORM \| EPOLLWRNORM \| EPOLLERR \| EPOLLHUP);
	725	+ /* Was that the last reader or writer, but not the other side? */
	726	+ if (!pipe->readers != !pipe->writers) {
	727	+ wake_up_interruptible_all(&pipe->rd_wait);
	728	+ wake_up_interruptible_all(&pipe->wr_wait);
601	729	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
602	730	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
603	731	}
..	..	@@ -626,27 +754,27 @@
626	754	return retval;
627	755	}
628	756
629		-static unsigned long account_pipe_buffers(struct user_struct *user,
630		- unsigned long old, unsigned long new)
	757	+unsigned long account_pipe_buffers(struct user_struct *user,
	758	+ unsigned long old, unsigned long new)
631	759	{
632	760	return atomic_long_add_return(new - old, &user->pipe_bufs);
633	761	}
634	762
635		-static bool too_many_pipe_buffers_soft(unsigned long user_bufs)
	763	+bool too_many_pipe_buffers_soft(unsigned long user_bufs)
636	764	{
637	765	unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft);
638	766
639	767	return soft_limit && user_bufs > soft_limit;
640	768	}
641	769
642		-static bool too_many_pipe_buffers_hard(unsigned long user_bufs)
	770	+bool too_many_pipe_buffers_hard(unsigned long user_bufs)
643	771	{
644	772	unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard);
645	773
646	774	return hard_limit && user_bufs > hard_limit;
647	775	}
648	776
649		-static bool is_unprivileged_user(void)
	777	+bool pipe_is_unprivileged_user(void)
650	778	{
651	779	return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
652	780	}
..	..	@@ -668,21 +796,24 @@
668	796
669	797	user_bufs = account_pipe_buffers(user, 0, pipe_bufs);
670	798
671		- if (too_many_pipe_buffers_soft(user_bufs) && is_unprivileged_user()) {
	799	+ if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) {
672	800	user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS);
673	801	pipe_bufs = PIPE_MIN_DEF_BUFFERS;
674	802	}
675	803
676		- if (too_many_pipe_buffers_hard(user_bufs) && is_unprivileged_user())
	804	+ if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user())
677	805	goto out_revert_acct;
678	806
679	807	pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
680	808	GFP_KERNEL_ACCOUNT);
681	809
682	810	if (pipe->bufs) {
683		- init_waitqueue_head(&pipe->wait);
	811	+ init_waitqueue_head(&pipe->rd_wait);
	812	+ init_waitqueue_head(&pipe->wr_wait);
684	813	pipe->r_counter = pipe->w_counter = 1;
685		- pipe->buffers = pipe_bufs;
	814	+ pipe->max_usage = pipe_bufs;
	815	+ pipe->ring_size = pipe_bufs;
	816	+ pipe->nr_accounted = pipe_bufs;
686	817	pipe->user = user;
687	818	mutex_init(&pipe->mutex);
688	819	return pipe;
..	..	@@ -700,13 +831,22 @@
700	831	{
701	832	int i;
702	833
703		- (void) account_pipe_buffers(pipe->user, pipe->buffers, 0);
	834	+#ifdef CONFIG_WATCH_QUEUE
	835	+ if (pipe->watch_queue)
	836	+ watch_queue_clear(pipe->watch_queue);
	837	+#endif
	838	+
	839	+ (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
704	840	free_uid(pipe->user);
705		- for (i = 0; i < pipe->buffers; i++) {
	841	+ for (i = 0; i < pipe->ring_size; i++) {
706	842	struct pipe_buffer *buf = pipe->bufs + i;
707	843	if (buf->ops)
708	844	pipe_buf_release(pipe, buf);
709	845	}
	846	+#ifdef CONFIG_WATCH_QUEUE
	847	+ if (pipe->watch_queue)
	848	+ put_watch_queue(pipe->watch_queue);
	849	+#endif
710	850	if (pipe->tmp_page)
711	851	__free_page(pipe->tmp_page);
712	852	kfree(pipe->bufs);
..	..	@@ -772,9 +912,19 @@
772	912	{
773	913	struct inode *inode = get_pipe_inode();
774	914	struct file *f;
	915	+ int error;
775	916
776	917	if (!inode)
777	918	return -ENFILE;
	919	+
	920	+ if (flags & O_NOTIFICATION_PIPE) {
	921	+ error = watch_queue_init(inode->i_pipe);
	922	+ if (error) {
	923	+ free_pipe_info(inode->i_pipe);
	924	+ iput(inode);
	925	+ return error;
	926	+ }
	927	+ }
778	928
779	929	f = alloc_file_pseudo(inode, pipe_mnt, "",
780	930	O_WRONLY \| (flags & (O_NONBLOCK \| O_DIRECT)),
..	..	@@ -796,6 +946,8 @@
796	946	}
797	947	res[0]->private_data = inode->i_pipe;
798	948	res[1] = f;
	949	+ stream_open(inode, res[0]);
	950	+ stream_open(inode, res[1]);
799	951	return 0;
800	952	}
801	953
..	..	@@ -804,7 +956,7 @@
804	956	int error;
805	957	int fdw, fdr;
806	958
807		- if (flags & ~(O_CLOEXEC \| O_NONBLOCK \| O_DIRECT))
	959	+ if (flags & ~(O_CLOEXEC \| O_NONBLOCK \| O_DIRECT \| O_NOTIFICATION_PIPE))
808	960	return -EINVAL;
809	961
810	962	error = create_pipe_files(files, flags);
..	..	@@ -881,12 +1033,52 @@
881	1033	return do_pipe2(fildes, 0);
882	1034	}
883	1035
	1036	+/*
	1037	+ * This is the stupid "wait for pipe to be readable or writable"
	1038	+ * model.
	1039	+ *
	1040	+ * See pipe_read/write() for the proper kind of exclusive wait,
	1041	+ * but that requires that we wake up any other readers/writers
	1042	+ * if we then do not end up reading everything (ie the whole
	1043	+ * "wake_next_reader/writer" logic in pipe_read/write()).
	1044	+ */
	1045	+void pipe_wait_readable(struct pipe_inode_info *pipe)
	1046	+{
	1047	+ pipe_unlock(pipe);
	1048	+ wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
	1049	+ pipe_lock(pipe);
	1050	+}
	1051	+
	1052	+void pipe_wait_writable(struct pipe_inode_info *pipe)
	1053	+{
	1054	+ pipe_unlock(pipe);
	1055	+ wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
	1056	+ pipe_lock(pipe);
	1057	+}
	1058	+
	1059	+/*
	1060	+ * This depends on both the wait (here) and the wakeup (wake_up_partner)
	1061	+ * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
	1062	+ * race with the count check and waitqueue prep.
	1063	+ *
	1064	+ * Normally in order to avoid races, you'd do the prepare_to_wait() first,
	1065	+ * then check the condition you're waiting for, and only then sleep. But
	1066	+ * because of the pipe lock, we can check the condition before being on
	1067	+ * the wait queue.
	1068	+ *
	1069	+ * We use the 'rd_wait' waitqueue for pipe partner waiting.
	1070	+ */
884	1071	static int wait_for_partner(struct pipe_inode_info pipe, unsigned int cnt)
885	1072	{
886		- int cur = *cnt;
	1073	+ DEFINE_WAIT(rdwait);
	1074	+ int cur = *cnt;
887	1075
888	1076	while (cur == *cnt) {
889		- pipe_wait(pipe);
	1077	+ prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
	1078	+ pipe_unlock(pipe);
	1079	+ schedule();
	1080	+ finish_wait(&pipe->rd_wait, &rdwait);
	1081	+ pipe_lock(pipe);
890	1082	if (signal_pending(current))
891	1083	break;
892	1084	}
..	..	@@ -895,7 +1087,7 @@
895	1087
896	1088	static void wake_up_partner(struct pipe_inode_info *pipe)
897	1089	{
898		- wake_up_interruptible(&pipe->wait);
	1090	+ wake_up_interruptible_all(&pipe->rd_wait);
899	1091	}
900	1092
901	1093	static int fifo_open(struct inode inode, struct file filp)
..	..	@@ -934,9 +1126,9 @@
934	1126	__pipe_lock(pipe);
935	1127
936	1128	/* We can only do regular read/write on fifos */
937		- filp->f_mode &= (FMODE_READ \| FMODE_WRITE);
	1129	+ stream_open(inode, filp);
938	1130
939		- switch (filp->f_mode) {
	1131	+ switch (filp->f_mode & (FMODE_READ \| FMODE_WRITE)) {
940	1132	case FMODE_READ:
941	1133	/*
942	1134	* O_RDONLY
..	..	@@ -958,7 +1150,7 @@
958	1150	}
959	1151	}
960	1152	break;
961		-
	1153	+
962	1154	case FMODE_WRITE:
963	1155	/*
964	1156	* O_WRONLY
..	..	@@ -978,7 +1170,7 @@
978	1170	goto err_wr;
979	1171	}
980	1172	break;
981		-
	1173	+
982	1174	case FMODE_READ \| FMODE_WRITE:
983	1175	/*
984	1176	* O_RDWR
..	..	@@ -1006,13 +1198,13 @@
1006	1198
1007	1199	err_rd:
1008	1200	if (!--pipe->readers)
1009		- wake_up_interruptible(&pipe->wait);
	1201	+ wake_up_interruptible(&pipe->wr_wait);
1010	1202	ret = -ERESTARTSYS;
1011	1203	goto err;
1012	1204
1013	1205	err_wr:
1014	1206	if (!--pipe->writers)
1015		- wake_up_interruptible(&pipe->wait);
	1207	+ wake_up_interruptible_all(&pipe->rd_wait);
1016	1208	ret = -ERESTARTSYS;
1017	1209	goto err;
1018	1210
..	..	@@ -1032,6 +1224,7 @@
1032	1224	.unlocked_ioctl = pipe_ioctl,
1033	1225	.release = pipe_release,
1034	1226	.fasync = pipe_fasync,
	1227	+ .splice_write = iter_file_splice_write,
1035	1228	};
1036	1229
1037	1230	/*
..	..	@@ -1051,20 +1244,91 @@
1051	1244	}
1052	1245
1053	1246	/*
	1247	+ * Resize the pipe ring to a number of slots.
	1248	+ *
	1249	+ * Note the pipe can be reduced in capacity, but only if the current
	1250	+ * occupancy doesn't exceed nr_slots; if it does, EBUSY will be
	1251	+ * returned instead.
	1252	+ */
	1253	+int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
	1254	+{
	1255	+ struct pipe_buffer *bufs;
	1256	+ unsigned int head, tail, mask, n;
	1257	+
	1258	+ bufs = kcalloc(nr_slots, sizeof(*bufs),
	1259	+ GFP_KERNEL_ACCOUNT \| __GFP_NOWARN);
	1260	+ if (unlikely(!bufs))
	1261	+ return -ENOMEM;
	1262	+
	1263	+ spin_lock_irq(&pipe->rd_wait.lock);
	1264	+ mask = pipe->ring_size - 1;
	1265	+ head = pipe->head;
	1266	+ tail = pipe->tail;
	1267	+
	1268	+ n = pipe_occupancy(head, tail);
	1269	+ if (nr_slots < n) {
	1270	+ spin_unlock_irq(&pipe->rd_wait.lock);
	1271	+ kfree(bufs);
	1272	+ return -EBUSY;
	1273	+ }
	1274	+
	1275	+ /*
	1276	+ * The pipe array wraps around, so just start the new one at zero
	1277	+ * and adjust the indices.
	1278	+ */
	1279	+ if (n > 0) {
	1280	+ unsigned int h = head & mask;
	1281	+ unsigned int t = tail & mask;
	1282	+ if (h > t) {
	1283	+ memcpy(bufs, pipe->bufs + t,
	1284	+ n * sizeof(struct pipe_buffer));
	1285	+ } else {
	1286	+ unsigned int tsize = pipe->ring_size - t;
	1287	+ if (h > 0)
	1288	+ memcpy(bufs + tsize, pipe->bufs,
	1289	+ h * sizeof(struct pipe_buffer));
	1290	+ memcpy(bufs, pipe->bufs + t,
	1291	+ tsize * sizeof(struct pipe_buffer));
	1292	+ }
	1293	+ }
	1294	+
	1295	+ head = n;
	1296	+ tail = 0;
	1297	+
	1298	+ kfree(pipe->bufs);
	1299	+ pipe->bufs = bufs;
	1300	+ pipe->ring_size = nr_slots;
	1301	+ if (pipe->max_usage > nr_slots)
	1302	+ pipe->max_usage = nr_slots;
	1303	+ pipe->tail = tail;
	1304	+ pipe->head = head;
	1305	+
	1306	+ spin_unlock_irq(&pipe->rd_wait.lock);
	1307	+
	1308	+ /* This might have made more room for writers */
	1309	+ wake_up_interruptible(&pipe->wr_wait);
	1310	+ return 0;
	1311	+}
	1312	+
	1313	+/*
1054	1314	* Allocate a new array of pipe buffers and copy the info over. Returns the
1055	1315	* pipe size if successful, or return -ERROR on error.
1056	1316	*/
1057	1317	static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
1058	1318	{
1059		- struct pipe_buffer *bufs;
1060		- unsigned int size, nr_pages;
1061	1319	unsigned long user_bufs;
	1320	+ unsigned int nr_slots, size;
1062	1321	long ret = 0;
1063	1322
1064		- size = round_pipe_size(arg);
1065		- nr_pages = size >> PAGE_SHIFT;
	1323	+#ifdef CONFIG_WATCH_QUEUE
	1324	+ if (pipe->watch_queue)
	1325	+ return -EBUSY;
	1326	+#endif
1066	1327
1067		- if (!nr_pages)
	1328	+ size = round_pipe_size(arg);
	1329	+ nr_slots = size >> PAGE_SHIFT;
	1330	+
	1331	+ if (!nr_slots)
1068	1332	return -EINVAL;
1069	1333
1070	1334	/*
..	..	@@ -1074,67 +1338,30 @@
1074	1338	* Decreasing the pipe capacity is always permitted, even
1075	1339	* if the user is currently over a limit.
1076	1340	*/
1077		- if (nr_pages > pipe->buffers &&
	1341	+ if (nr_slots > pipe->max_usage &&
1078	1342	size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
1079	1343	return -EPERM;
1080	1344
1081		- user_bufs = account_pipe_buffers(pipe->user, pipe->buffers, nr_pages);
	1345	+ user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);
1082	1346
1083		- if (nr_pages > pipe->buffers &&
	1347	+ if (nr_slots > pipe->max_usage &&
1084	1348	(too_many_pipe_buffers_hard(user_bufs) \|\|
1085	1349	too_many_pipe_buffers_soft(user_bufs)) &&
1086		- is_unprivileged_user()) {
	1350	+ pipe_is_unprivileged_user()) {
1087	1351	ret = -EPERM;
1088	1352	goto out_revert_acct;
1089	1353	}
1090	1354
1091		- /*
1092		- * We can shrink the pipe, if arg >= pipe->nrbufs. Since we don't
1093		- * expect a lot of shrink+grow operations, just free and allocate
1094		- * again like we would do for growing. If the pipe currently
1095		- * contains more buffers than arg, then return busy.
1096		- */
1097		- if (nr_pages < pipe->nrbufs) {
1098		- ret = -EBUSY;
	1355	+ ret = pipe_resize_ring(pipe, nr_slots);
	1356	+ if (ret < 0)
1099	1357	goto out_revert_acct;
1100		- }
1101	1358
1102		- bufs = kcalloc(nr_pages, sizeof(*bufs),
1103		- GFP_KERNEL_ACCOUNT \| __GFP_NOWARN);
1104		- if (unlikely(!bufs)) {
1105		- ret = -ENOMEM;
1106		- goto out_revert_acct;
1107		- }
1108		-
1109		- /*
1110		- * The pipe array wraps around, so just start the new one at zero
1111		- * and adjust the indexes.
1112		- */
1113		- if (pipe->nrbufs) {
1114		- unsigned int tail;
1115		- unsigned int head;
1116		-
1117		- tail = pipe->curbuf + pipe->nrbufs;
1118		- if (tail < pipe->buffers)
1119		- tail = 0;
1120		- else
1121		- tail &= (pipe->buffers - 1);
1122		-
1123		- head = pipe->nrbufs - tail;
1124		- if (head)
1125		- memcpy(bufs, pipe->bufs + pipe->curbuf, head * sizeof(struct pipe_buffer));
1126		- if (tail)
1127		- memcpy(bufs + head, pipe->bufs, tail * sizeof(struct pipe_buffer));
1128		- }
1129		-
1130		- pipe->curbuf = 0;
1131		- kfree(pipe->bufs);
1132		- pipe->bufs = bufs;
1133		- pipe->buffers = nr_pages;
1134		- return nr_pages * PAGE_SIZE;
	1359	+ pipe->max_usage = nr_slots;
	1360	+ pipe->nr_accounted = nr_slots;
	1361	+ return pipe->max_usage * PAGE_SIZE;
1135	1362
1136	1363	out_revert_acct:
1137		- (void) account_pipe_buffers(pipe->user, nr_pages, pipe->buffers);
	1364	+ (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
1138	1365	return ret;
1139	1366	}
1140	1367
..	..	@@ -1143,9 +1370,17 @@
1143	1370	* location, so checking ->i_pipe is not enough to verify that this is a
1144	1371	* pipe.
1145	1372	*/
1146		-struct pipe_inode_info get_pipe_info(struct file file)
	1373	+struct pipe_inode_info get_pipe_info(struct file file, bool for_splice)
1147	1374	{
1148		- return file->f_op == &pipefifo_fops ? file->private_data : NULL;
	1375	+ struct pipe_inode_info *pipe = file->private_data;
	1376	+
	1377	+ if (file->f_op != &pipefifo_fops \|\| !pipe)
	1378	+ return NULL;
	1379	+#ifdef CONFIG_WATCH_QUEUE
	1380	+ if (for_splice && pipe->watch_queue)
	1381	+ return NULL;
	1382	+#endif
	1383	+ return pipe;
1149	1384	}
1150	1385
1151	1386	long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
..	..	@@ -1153,7 +1388,7 @@
1153	1388	struct pipe_inode_info *pipe;
1154	1389	long ret;
1155	1390
1156		- pipe = get_pipe_info(file);
	1391	+ pipe = get_pipe_info(file, false);
1157	1392	if (!pipe)
1158	1393	return -EBADF;
1159	1394
..	..	@@ -1164,7 +1399,7 @@
1164	1399	ret = pipe_set_size(pipe, arg);
1165	1400	break;
1166	1401	case F_GETPIPE_SZ:
1167		- ret = pipe->buffers * PAGE_SIZE;
	1402	+ ret = pipe->max_usage * PAGE_SIZE;
1168	1403	break;
1169	1404	default:
1170	1405	ret = -EINVAL;
..	..	@@ -1186,16 +1421,20 @@
1186	1421	* any operations on the root directory. However, we need a non-trivial
1187	1422	* d_name - pipe: will go nicely and kill the special-casing in procfs.
1188	1423	*/
1189		-static struct dentry pipefs_mount(struct file_system_type fs_type,
1190		- int flags, const char dev_name, void data)
	1424	+
	1425	+static int pipefs_init_fs_context(struct fs_context *fc)
1191	1426	{
1192		- return mount_pseudo(fs_type, "pipe:", &pipefs_ops,
1193		- &pipefs_dentry_operations, PIPEFS_MAGIC);
	1427	+ struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC);
	1428	+ if (!ctx)
	1429	+ return -ENOMEM;
	1430	+ ctx->ops = &pipefs_ops;
	1431	+ ctx->dops = &pipefs_dentry_operations;
	1432	+ return 0;
1194	1433	}
1195	1434
1196	1435	static struct file_system_type pipe_fs_type = {
1197	1436	.name = "pipefs",
1198		- .mount = pipefs_mount,
	1437	+ .init_fs_context = pipefs_init_fs_context,
1199	1438	.kill_sb = kill_anon_super,
1200	1439	};
1201	1440